diff --git a/Documentation/ABI/testing/debugfs-ub-hisi-ubus b/Documentation/ABI/testing/debugfs-ub-hisi-ubus new file mode 100644 index 0000000000000000000000000000000000000000..69ba558bfc025c226bbb0c72ce3be7158f4ddd20 --- /dev/null +++ b/Documentation/ABI/testing/debugfs-ub-hisi-ubus @@ -0,0 +1,37 @@ +What: /sys/kernel/debug/UB_BUS_CTL/eu_table +Date: Oct 2025 +Contact: Junlong Zheng +Description: Display the contents of the EID-UPI entry. + By default, the EID and UPI key-value pair for entry 0 is displayed. + By writing an entry index to the properties file, you can retrieve + the content of the corresponding entry. + + Example:: + + Display the content of entry5: + # echo 5 > /sys/kernel/debug/UB_BUS_CTL/eu_table + # cat /sys/kernel/debug/UB_BUS_CTL/eu_table + +What: /sys/kernel/debug/UB_BUS_CTL/hi_msgq-/reg_info +Date: Oct 2025 +Contact: Junlong Zheng +Description: Display the register information for the specified queue of the designated + UB Bus controller. + +What: /sys/kernel/debug/UB_BUS_CTL/hi_msgq-/q_entry_info +Date: Oct 2025 +Contact: Junlong Zheng +Description: Display the SQE and CQE contents of the specified MSQG for the designated + UB Bus controller. By default, the content of SQ entry 0 is displayed. + By writing the queue type and entry index to the properties file, you can + retrieve the content of the corresponding entry. + + Example:: + + Output the content of SQ entry3: + # echo 0 3 > /sys/kernel/debug/UB_BUS_CTL/hi_msgq-/q_entry_info + # cat /sys/kernel/debug/UB_BUS_CTL/hi_msgq-/q_entry_info + + Output the content of CQ entry5: + # echo 2 5 > /sys/kernel/debug/UB_BUS_CTL/hi_msgq-/q_entry_info + # cat /sys/kernel/debug/UB_BUS_CTL/hi_msgq-/q_entry_info \ No newline at end of file diff --git a/Documentation/ABI/testing/sysfs-bus-ub b/Documentation/ABI/testing/sysfs-bus-ub new file mode 100644 index 0000000000000000000000000000000000000000..f7b3193958b7735718176f372b424fda29102326 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-ub @@ -0,0 +1,427 @@ +What: /sys/bus/ub/cluster +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Indicates the current system operating mode: + 1 for cluster mode, 0 for Standalone mode. + +What: /sys/bus/ub/instance +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the list of bus instances created in the current system. + By default, it starts from the first one. Writing numbers into + the file can change the starting position of the output bus + instance. + +What: /sys/bus/ub/drivers/.../bind +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Writing an entity number to this file will cause the driver + to attempt to bind to the entity. This is useful for + overriding default bindings. The entity number is + the same as found in /sys/bus/ub/devices/. + For example:: + + # echo 00002 > /sys/bus/ub/drivers/sample/bind + +What: /sys/bus/ub/drivers/.../unbind +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Writing an entity number to this file will cause the + driver to attempt to unbind from the entity. This may be + useful when overriding default bindings. The entity + number is the same as found in /sys/bus/ub/devices/. + For example:: + + # echo 00002 > /sys/bus/ub/drivers/sample/unbind + +What: /sys/bus/ub/drivers/.../new_id +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Writing a device ID to this file will attempt to + dynamically add a new device ID to a UB device driver. + This may allow the driver to support more hardware than + was included in the driver's static device ID support + table at compile time. The format for the device ID is: + VVVV DDDD MVVV MMMM CCCC MMMM PPPP. That is Vendor ID, + Device ID, Module Vendor ID, Module ID, Class, Class Mask + and Private Driver Data. The Vendor ID and Device ID fields + are required, the rest are optional. Upon successfully + adding an ID, the driver will probe for the device and + attempt to bind to it. + For example:: + + # echo cc08 a001 > /sys/bus/ub/drivers/sample/new_id + +What: /sys/bus/ub/drivers/.../remove_id +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Writing a device ID to this file will remove an ID + that was dynamically added via the new_id sysfs entry. + The format for the device ID is: + VVVV DDDD MVVV MMMM CCCC MMMM. That is Vendor ID, Device + ID, Module Vendor ID, Module ID, Class and Class Mask. + The Vendor ID and Device ID fields are required, the rest + are optional. After successfully removing an ID, + the driver will no longer support the device. + This is useful to ensure auto probing won't + match the driver to the device. + For example:: + + # echo cc08 a001 > /sys/bus/ub/drivers/sample/remove_id + +What: /sys/bus/ub/devices/.../class_code +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the class code type of the entity. + +What: /sys/bus/ub/devices/.../config +Date: Oct 2025 +Contact: Junlong Zheng +Description: + A channel is provided for user-mode programs to access the + entity configuration space. User programs can open the file + using the open system call and then perform read/write + operations on the configuration space using the pread/pwrite + system calls. + For details, please refer to the implementation of ubutils + . + +What: /sys/bus/ub/devices/.../device +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the Device ID of the entity. + +What: /sys/bus/ub/devices/.../device_reset +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Writing 1 to this file can trigger a device-level reset. All + entities below it will be reset. + Supported only by Entity0. + +What: /sys/bus/ub/devices/.../direct_link +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the link connection relationships and the peer + information of the ports connected to this entity. + +What: /sys/bus/ub/devices/.../driver_override +Date: Oct 2025 +Contact: Junlong Zheng +Description: + This file allows the driver for a device to be specified which + will override standard static and dynamic ID matching. When + specified, only a driver with a name matching the value written + to driver_override will have an opportunity to bind to the + device. The override is specified by writing a string to the + driver_override file (echo sample > driver_override) and + may be cleared with an empty string (echo > driver_override). + This returns the device to standard matching rules binding. + Writing to driver_override does not automatically unbind the + device from its current driver or make any attempt to + automatically load the specified driver. If no driver with a + matching name is currently loaded in the kernel, the device + will not bind to any driver. This also allows devices to + opt-out of driver binding using a driver_override name such as + "none". Only a single driver may be specified in the override, + there is no support for parsing delimiters. + +What: /sys/bus/ub/devices/.../eid +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the entity's EID. + +What: /sys/bus/ub/devices/.../entity_idx +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the index of the entity, numbered starting from 0. + +What: /sys/bus/ub/devices/.../guid +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the GUID of the entity. + +What: /sys/bus/ub/devices/.../instance +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the EID of the bus instance bound to the entity. + +What: /sys/bus/ub/devices/.../kref +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the current reference count of the entity. + +What: /sys/bus/ub/devices/.../match_driver +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Indicates whether the current entity is ready for driver + matching. Some entities require additional initialization work, + so this entry is provided to control the entity separately. + In this case, it is necessary to ensure a certain timing + sequence; For example, the driver should be loaded only after + this status of the entity is set to 1 to ensure that the driver + probe is correctly initiated. + +What: /sys/bus/ub/devices/.../numa +Date: Oct 2025 +Contact: Junlong Zheng +Description: + This file contains the NUMA node to which the UB Entity is + attached, or -1 if the node is unknown. The initial value + comes from UBRT table, defined in the UB firmware specification. + +What: /sys/bus/ub/devices/.../primary_cna +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the primary compact network address of the entity. + +What: /sys/bus/ub/devices/.../primary_entity +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the entity number of the entity0 to which this entity + belongs. + +What: /sys/bus/ub/devices/.../mue_list +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display a list of all MUEs under this entity, excluding itself. + Only Entity0 has this attribute file. + +What: /sys/bus/ub/devices/.../reset +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Writing 1 to this file can trigger an entity-level reset, only + reset this entity, it will not affect other entities. + +What: /sys/bus/ub/devices/.../resource +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Information about the resource space of the entity is displayed, + with a total of 3 entries, each consisting of the following + three components: start_address, end_address, flags. + If all values are 0, it indicates that the resource space is + not supported. + +What: /sys/bus/ub/devices/.../resource +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Provide attribute files to the user-mode driver. Through the + open and mmap system calls, the resource space of an entity can + be mapped into the process space for direct access, thereby + improving the efficiency of cross-mode resource space access. + The memory attribute mapped by this interface is the device + attribute. + +What: /sys/bus/ub/devices/.../resource_wc +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Provide attribute files to the user-mode driver. Through the + open and mmap system calls, the resource space of an entity can + be mapped into the process space for direct access, thereby + improving the efficiency of cross-mode resource space access. + The memory attribute mapped by this interface is the + write-combine attribute. + +What: /sys/bus/ub/devices/.../sw_cap +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display whether forwarding capability is supported. + Only UBC supports it. + +What: /sys/bus/ub/devices/.../tid +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the Token ID of the entity. The entity uses this + Token ID to access system memory. + +What: /sys/bus/ub/devices/.../type +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the type of the entity. The type is a subdomain segment + of GUID. + +What: /sys/bus/ub/devices/.../ubc +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the entity number of the UB controller associated with + the entity. + +What: /sys/bus/ub/devices/.../ub_numues +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the number of UEs that have been enabled for this + entity. Writing a value to the file enables the UEs. The written + value must be within the range of ub_totalues. Writing 0 + disables all UEs. + Only MUE supports this file. + +What: /sys/bus/ub/devices/.../ub_total_entities +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the number of all entities supported by this entity. + Only Entity0 supports this file. + +What: /sys/bus/ub/devices/.../ub_totalues +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the number of UEs owned by this entity. + Only MUE supports this file. + +What: /sys/bus/ub/devices/.../ue_list +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display a list of all UEs under this entity, excluding itself. + Only MUE has this attribute file. + +What: /sys/bus/ub/devices/.../upi +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display UPI of the entity. + +What: /sys/bus/ub/devices/.../vendor +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display Vendor ID of the entity. + +What: /sys/bus/ub/devices/.../msi_irqs/ +Date: Oct 2025 +Contact: Junlong Zheng +Description: + The /sys/bus/ub/devices/.../msi_irqs/ directory contains a + variable set of files, with each file being named after a + corresponding msi irq vector allocated to that entity. + +What: /sys/bus/ub/devices/.../msi_irqs/ +Date: Oct 2025 +Contact: Junlong Zheng +Description: + This attribute indicates the mode that the irq vector named by + the file is in (msi vs. msix) + +What: /sys/bus/ub/devices/.../port/asy_link_width +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Indicates whether the port supports asymmetric link width. + Supported only on physical port. + + +What: /sys/bus/ub/devices/.../port/boundary +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Indicates whether the port is a boundary port. + +What: /sys/bus/ub/devices/.../port/cna +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display the CNA of this port. + +What: /sys/bus/ub/devices/.../port/glb_qdlws +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Write 1 to enable global dynamic lane adjustment, + write 0 to disable this function. + Supported only on physical port. + +What: /sys/bus/ub/devices/.../port/linkup +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Display whether the port has established a connection. + +What: /sys/bus/ub/devices/.../port/neighbor +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Displays the entity number of the peer entity at the port. + If no link is established, it displays "No Neighbor". + +What: /sys/bus/ub/devices/.../port/neighbor_guid +Date: Oct 2025 +Contact: Junlong Zheng +Description: + If there is a peer entity, display the GUID of the peer entity. + Otherwise, display "No Neighbor". + +What: /sys/bus/ub/devices/.../port/neighbor_port_idx +Date: Oct 2025 +Contact: Junlong Zheng +Description: + If there is a peer entity, the port index of the peer entity is + displayed. Otherwise, display "No Neighbor". + +What: /sys/bus/ub/devices/.../port/port_reset +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Supports individual port reset, triggered by writing 1. + Supported only on physical port. + +What: /sys/bus/ub/devices/.../port/qdlws_exec_state +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Obtain the hardware execution status of the current dynamically + adjustable lane. + Supported only on physical port. + +What: /sys/bus/ub/devices/.../port/rx_qdlws +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Enable/Disable dynamic lane adjustment in the RX direction. + Supported only on physical port. + +What: /sys/bus/ub/devices/.../port/tx_qdlws +Date: Oct 2025 +Contact: Junlong Zheng +Description: + Enable/Disable dynamic lane adjustment in the TX direction. + Supported only on physical port. + +What: /sys/bus/ub/devices/.../slot/power +Date: Oct 2025 +Contact: Junlong Zheng +Description: + This feature supports hot-plug notification. + Display the current slot status, the value can be "on", + "poweron", "poweroff", "off" or "unknown state". And can + write 1 to enable power on the slot, and write 0 to + power it off. + This file is supported only by entities that support + hot-plug features. diff --git a/Documentation/ABI/testing/sysfs-class-iommu-ummu-bypass-mpam b/Documentation/ABI/testing/sysfs-class-iommu-ummu-bypass-mpam new file mode 100644 index 0000000000000000000000000000000000000000..c28753fb8b7da59c1f927490fcc95aa261087c3b --- /dev/null +++ b/Documentation/ABI/testing/sysfs-class-iommu-ummu-bypass-mpam @@ -0,0 +1,31 @@ +What: /sys/class/iommu/ummu./ummu_bypass_mpam/bp_partid +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + The partID value used by the MPAM function in the bypass UMMU + scenario. Format: %x. + +What: /sys/class/iommu/ummu./ummu_bypass_mpam/bp_pmg +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + The PMG value used by the MPAM function in the bypass UMMU scenario. + Format: %x. + +What: /sys/class/iommu/ummu./ummu_bypass_mpam/bp_run +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + After setting the bp_partid and bp_pmg values, write 1 to bp_run to + apply these values to the UMMU device. These values define the IO + regions that bypass the UMMU device in the bypass UMMU scenario. + +What: /sys/class/iommu/ummu./ummu_bypass_mpam/bp_mpam_info +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + Retrieve the currently active MPAM configuration from the UMMU device. diff --git a/Documentation/ABI/testing/sysfs-class-iommu-ummu-iommu b/Documentation/ABI/testing/sysfs-class-iommu-ummu-iommu new file mode 100644 index 0000000000000000000000000000000000000000..48ba4d6d4c60db8b81660ac6ec90f7880feb8783 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-class-iommu-ummu-iommu @@ -0,0 +1,113 @@ +What: /sys/class/iommu/ummu./ummu-iommu/eid_list +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + List of all EIDs registered to UMMU. + +What: /sys/class/iommu/ummu./ummu-iommu/evtq_log2num +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + The number of Event Queues in the non-secure state of the + UMMU device (in log2). + +What: /sys/class/iommu/ummu./ummu-iommu/evtq_log2size +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + The depth of each Event Queue in the non-secure state of the + UMMU device (in log2). + +What: /sys/class/iommu/ummu./ummu-iommu/features +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + UMMU device capabilities. + +What: /sys/class/iommu/ummu./ummu-iommu/mcmdq_log2num +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + The number of Command Queues in the non-secure state of the + UMMU device in kernel mode. + +What: /sys/class/iommu/ummu./ummu-iommu/mcmdq_log2size +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + The depth of each command queue in the non-secure state of the + UMMU device in kernel mode. + +What: /sys/class/iommu/ummu./ummu-iommu/permq_ent_num +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + The depth of the permission queue in user mode of the UMMU + device (in log2). + +What: /sys/class/iommu/ummu./ummu-iommu/permq_num +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + The number of permission queues in user mode of the UMMU + device (in log2). + +What: /sys/class/iommu/ummu./ummu-iommu/ias +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + The bit width of the input address supported by the UMMU + device. + +What: /sys/class/iommu/ummu./ummu-iommu/oas +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + The bit width of the output address of the UMMU device. + +What: /sys/class/iommu/ummu./ummu-iommu/options +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + Configurable features of the UMMU device. + +What: /sys/class/iommu/ummu./ummu-iommu/pgsize_bitmap +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + Supported page size bitmap of the UMMU translation table. + +What: /sys/class/iommu/ummu./ummu-iommu/ptsize_bitmap +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + Supported page size bitmap of the UMMU MAPT table + (permission check). + +What: /sys/class/iommu/ummu./ummu-iommu/tid_bits +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + Maximum TokenID bit width supported in non-secure state. + +What: /sys/class/iommu/ummu./ummu-iommu/tid_type +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + The domain_type corresponding to the TokenID, which requires + the TokenID value as input. Format: %x. diff --git a/Documentation/ABI/testing/sysfs-class-iommu-ummu-uotr-mpam b/Documentation/ABI/testing/sysfs-class-iommu-ummu-uotr-mpam new file mode 100644 index 0000000000000000000000000000000000000000..8bbe9d65c7d9afa276f42f464231dc8864f6b0f3 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-class-iommu-ummu-uotr-mpam @@ -0,0 +1,31 @@ +What: /sys/class/iommu/ummu./ummu_uotr_mpam/uotr_partid +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + The PartID value used by the MPAM function to tag UMMU-initiated + traffic. Format: %x. + +What: /sys/class/iommu/ummu./ummu_uotr_mpam/uotr_pmg +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + The PMG value used by the MPAM function to tag UMMU-initiated traffic. + Format: %x. + +What: /sys/class/iommu/ummu./ummu_uotr_mpam/uotr_run +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + After setting the uotr_partid and uotr_pmg values, write 1 to uotr_run + to apply them to the UMMU device. These values tag I/O traffic initiated + by the UMMU itself. + +What: /sys/class/iommu/ummu./ummu_uotr_mpam/uotr_mpam_info +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Jingbin Wu +Description: + Retrieve the MPAM configuration last applied to the UMMU device. diff --git a/Documentation/ABI/testing/sysfs-devices-platform-ummu_vdev b/Documentation/ABI/testing/sysfs-devices-platform-ummu_vdev new file mode 100644 index 0000000000000000000000000000000000000000..2812b512c20e14177bba694fe26721f1ecf09ab4 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-devices-platform-ummu_vdev @@ -0,0 +1,19 @@ +What: /sys/devices/platform/ummu_tid_root/logic_ummu/ummu_vdev..auto/ummu-vdev-attr/tid_mode +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Yanlong Zhu +Description: + (RO) Displays the MAPT mode of Token ID. Format: %d. Allowed values: + + == =========================================== + 0 Token ID is table mode. + 1 Token ID is entry mode. + 2 Fatal error occurred. + == =========================================== + +What: /sys/devices/platform/ummu_tid_root/logic_ummu/ummu_vdev..auto/ummu-vdev-attr/tid_val +Date: Oct 2025 +KernelVersion: 6.6 +Contact: Yanlong Zhu +Description: + (RO) Displays the value of Token ID. Format: %u. diff --git a/Documentation/admin-guide/perf/ummu-pmu.rst b/Documentation/admin-guide/perf/ummu-pmu.rst new file mode 100644 index 0000000000000000000000000000000000000000..5447f8c61ac66212b5a0d0a5319a61760b01d6f8 --- /dev/null +++ b/Documentation/admin-guide/perf/ummu-pmu.rst @@ -0,0 +1,112 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +====================================== +UMMU Performance Monitoring Unit (PMU) +====================================== + +The UMMU includes a Performance Monitor Unit (PMU) to track and collect +statistics on key hardware events, such as TLB/PLB cache hit rates and +lookup latencies. By leveraging the Linux kernel's perf subsystem, the +collected event data can be efficiently processed, analyzed, and +visualized to drive targeted optimizations for UMMU performance. + +Usage +===== + +Basic usage follows the standard Linux kernel perf interface. The UMMU +device supports the following PMU events, which are exposed under the perf +event directory: + +.. code-block:: bash + + ls -l /sys/bus/event_source/devices/ummu_pmcg_0/events + +Constraints +=========== + +- No more than 8 events can be monitored at the same time. + +UMMU PMU Events +=============== + +.. table:: PMU Events in UMMU and Their Meanings + + +---------------------------------+-------------------------------------------------------------+ + | Event | Meaning | + +=================================+=============================================================+ + | kv_table_rd_average_latency | Average bus latency for reading key-value (KV) and | + | | content-addressable memory (CAM) tables during the | + | | conversion from DstEID to tecte_tag. | + +---------------------------------+-------------------------------------------------------------+ + | swif_cmd_send_num | Command count generated by SWIF. | + +---------------------------------+-------------------------------------------------------------+ + | swif_dvm_sync_latency | Average latency during execution of Sync commands issued | + | | by SWIF DVM. | + +---------------------------------+-------------------------------------------------------------+ + | swif_kcmd_ns_sync_latency | Average latency during execution of Sync commands issued | + | | by the SWIF KCMD non-secure queue. | + +---------------------------------+-------------------------------------------------------------+ + | swif_kcmd_s_sync_latency | Average latency during execution of Sync commands issued | + | | by the SWIF KCMD secure queue. | + +---------------------------------+-------------------------------------------------------------+ + | swif_ucmd_sync_latency | Average latency during execution of Sync commands issued | + | | by SWIF UCMD. | + +---------------------------------+-------------------------------------------------------------+ + | tbu_plb_cache_hit_rate | The hit rate observed during table lookups in the TBU PLB | + | | table. | + +---------------------------------+-------------------------------------------------------------+ + | tbu_tlb_cache_hit_rate | The hit rate observed during table lookups in the TBU TLB | + | | table. | + +---------------------------------+-------------------------------------------------------------+ + | tcu_cntx_cache_miss_num | The number of cache misses observed in the TCU context | + | | cache. | + +---------------------------------+-------------------------------------------------------------+ + | tcu_gpc_cache_hit_rate | The hit rate observed during access operations in the | + | | TCU GPC cache. | + +---------------------------------+-------------------------------------------------------------+ + | tcu_gpc_req_latency | The latency observed during GPC lookup operations in the | + | | TCU GPC module. | + +---------------------------------+-------------------------------------------------------------+ + | tcu_pptw_cache_hit_rate | The hit rate observed during access operations in the | + | | TCU PPTW cache. | + +---------------------------------+-------------------------------------------------------------+ + | tcu_pptw_req_latency | The latency observed during permission-based PTW lookup | + | | operations in the TCU PPTW module. | + +---------------------------------+-------------------------------------------------------------+ + | tcu_pptw_req_num | PPTW Request Count. | + +---------------------------------+-------------------------------------------------------------+ + | tcu_tptw_cache_hit_rate | The hit rate observed during access operations in the | + | | TCU TPTW cache. | + +---------------------------------+-------------------------------------------------------------+ + | tcu_tptw_req_latency | The latency observed during PTW lookup operations in the | + | | TCU TPTW module. | + +---------------------------------+-------------------------------------------------------------+ + | ubif_kv_cache_hit_rate | The hit rate observed during access operations in the | + | | UBIF KV cache. | + +---------------------------------+-------------------------------------------------------------+ + | ummu_req_average_latency | The average latency of table lookup requests during system | + | | operation. | + +---------------------------------+-------------------------------------------------------------+ + | ummu_req_rate | The rate of table lookup requests during system operation. | + +---------------------------------+-------------------------------------------------------------+ + | ummu_rsp_rate | The rate of table lookup results during system operation. | + +---------------------------------+-------------------------------------------------------------+ + | tbu_ptw_pack_rate | The rate of address translation table lookup requests sent | + | | by TBU RAB to TCU. | + +---------------------------------+-------------------------------------------------------------+ + | tbu_pptw_pack_rate | The rate of permission table lookup requests sent by TBU | + | | RAB to TCU. | + +---------------------------------+-------------------------------------------------------------+ + | tbu_ptw_latency | Average end-to-end latency of PTW requests from TBU RAB. | + +---------------------------------+-------------------------------------------------------------+ + | tbu_pptw_latency | Average end-to-end latency of PPTW requests from TBU RAB. | + +---------------------------------+-------------------------------------------------------------+ + | tbu_rab_buf_use_rate | Buffer utilization rate of TBU RAB. | + +---------------------------------+-------------------------------------------------------------+ + | swif_kcmd_gpc_sync_latency | Average execution latency of Sync commands issued by the | + | | SWIF KCMD GPC queue, excluding those via the DVM interface. | + +---------------------------------+-------------------------------------------------------------+ + | swif_kcmd_realm_sync_latency | Average execution latency of Sync commands issued by the | + | | SWIF KCMD REALM queue, excluding those via the DVM | + | | interface. | + +---------------------------------+-------------------------------------------------------------+ diff --git a/Documentation/admin-guide/tainted-kernels.rst b/Documentation/admin-guide/tainted-kernels.rst index 92a8a07f5c438c48a8df6fdef51050284074820d..6a63aede0d864e48ec68a446ccb047db3d5187a5 100644 --- a/Documentation/admin-guide/tainted-kernels.rst +++ b/Documentation/admin-guide/tainted-kernels.rst @@ -101,6 +101,7 @@ Bit Log Number Reason that got the kernel tainted 16 _/X 65536 auxiliary taint, defined for and used by distros 17 _/T 131072 kernel was built with the struct randomization plugin 18 _/N 262144 an in-kernel test has been run + 19 _/J 524288 userspace used a mutating debug operation in fwctl === === ====== ======================================================== Note: The character ``_`` is representing a blank in this table to make reading @@ -182,3 +183,8 @@ More detailed explanation for tainting produce extremely unusual kernel structure layouts (even performance pathological ones), which is important to know when debugging. Set at build time. + + 19) ``J`` if userpace opened /dev/fwctl/* and performed a FWTCL_RPC_DEBUG_WRITE + to use the devices debugging features. Device debugging features could + cause the device to malfunction in undefined ways. + diff --git a/Documentation/devicetree/bindings/iommu/hisi,ummu.yaml b/Documentation/devicetree/bindings/iommu/hisi,ummu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61d0074f7c4c93aa10f1e75ee7093f27e4a407fa --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/hisi,ummu.yaml @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iommu/hisi,ummu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: HiSilicon UMMU Architecture Implementation + +maintainers: + - Jingbin Wu + +description: |+ + UMMU is an IOMMU device that performs address translation and permission checking + using DstEID, TokenID, and UBA as input parameters. + +properties: + $nodename: + pattern: "^ummu@[0-9a-f]*" + compatible: + const: ub,ummu + index: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + UMMU device index. Used to identify a specific UMMU instance in systems + with multiple UMMU devices. + msi-parent: + $ref: /schemas/types.yaml#/definitions/phandle + description: | + MSI parent device phandle. Required for MSI interrupt handling. + +required: + - compatible + - index + - msi-parent + +additionalProperties: false + +examples: + - |+ + ummu@0 { + compatible = "ub,ummu"; + index = <0x0>; + msi-parent = <&its>; + }; diff --git a/Documentation/devicetree/bindings/perf/hisi,ummu-pmu.yaml b/Documentation/devicetree/bindings/perf/hisi,ummu-pmu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c16fad1c35fef8c426f443bfaa798d4a56aa43c3 --- /dev/null +++ b/Documentation/devicetree/bindings/perf/hisi,ummu-pmu.yaml @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/perf/hisi,ummu-pmu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: HiSilicon UMMU Performance Monitor Unit (PMU) + +maintainers: + - Jingbin Wu + +description: | + The UMMU includes a PMU(Performance Monitor Unit ) to monitor and collect + statistics on key hardware events, such as TLB/PLB cache hit rates and + lookup latencies. + +properties: + $nodename: + pattern: "^ummu-pmu@[0-9a-f]*" + compatible: + const: ub,ummu_pmu + index: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + PMU device index. Identifies a specific UMMU-PMU instance in multi-UMMU-PMU + systems. + msi-parent: + $ref: /schemas/types.yaml#/definitions/phandle + description: | + MSI parent device phandle. Required for MSI interrupt handling. + +required: + - compatible + - index + - msi-parent + +additionalProperties: false + +examples: + - | + ummu-pmu@0 { + compatible = "ub,ummu_pmu"; + index = <0x0>; + msi-parent = <&its>; + }; diff --git a/Documentation/devicetree/bindings/ub/hisi,ubc.yaml b/Documentation/devicetree/bindings/ub/hisi,ubc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2219dd7902c1b075b2ac994ba6418f4c02ebbe90 --- /dev/null +++ b/Documentation/devicetree/bindings/ub/hisi,ubc.yaml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/ub/hisi,ubc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: HiSilicon UBC (Unified Bus Controller) platform device + +maintainers: + - Yuhao Xiang + +description: |+ + This platform device was added to enable the automatic loading of the + hisi_ubus driver. If this feature is not needed, you can omit adding this + device and manually load the driver instead. + +properties: + $nodename: + pattern: "^hisi-ubc$" + description: | + The node name should be "hisi-ubc". + + compatible: + const: "hisi,ubc" + description: | + The compatible property should be "hisi,ubc" to identify the device as a + HiSilicon UBC platform device. + +unevaluatedProperties: false + +examples: + - |+ + hisi-ubc { + compatible = "hisi,ubc"; + }; diff --git a/Documentation/devicetree/bindings/ub/ub,ubc.yaml b/Documentation/devicetree/bindings/ub/ub,ubc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..012a293cf9e2532f13f5926792e7fab820e6fe3b --- /dev/null +++ b/Documentation/devicetree/bindings/ub/ub,ubc.yaml @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/ub/ub,ubc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: UBC (Unified Bus Controller) platform device + +maintainers: + - Yuhao Xiang + +description: | + The UBC platform device reported UBC interrupt number and the association + between the ubc and the interrupt controller. + +properties: + $nodename: + pattern: "^ubc@[0-9a-f]*" + + compatible: + const: "ub,ubc" + + interrupts: + maxItems: 1 + description: | + The interrupt specifier for the UBC. Used by the msgq of the ub controller. + + index: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + The index of the UBC. This is used to identify the specific UBC + in a system with multiple UBC devices. Starts from 0. + + msi-parent: + description: The msi interrupt for the UBC. Used by the ub entity connected + to UBC. + +required: + - compatible + - interrupts + - index + - msi-parent + +unevaluatedProperties: true + +examples: + - |+ + ubc@0 { + compatible = "ub,ubc"; + #interrupt-cells = <0x3>; + interrupt-parent = <0x01>; + interrupts = <0x00 0xcb 0x4>; + index = <0x00>; + msi-parent = <&its 0x54c0>; + }; diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst index f0f8f521f65bcf70959f275b1a192c8a26b55420..f2c5d67d47e96e797d21d7513ea75797122727f7 100644 --- a/Documentation/driver-api/index.rst +++ b/Documentation/driver-api/index.rst @@ -115,6 +115,7 @@ available subsections can be seen below. hte/index wmi crypto/index + ub/index .. only:: subproject and html diff --git a/Documentation/driver-api/ub/cdma.rst b/Documentation/driver-api/ub/cdma.rst new file mode 100644 index 0000000000000000000000000000000000000000..784962a71af5ef4010b4ca59abef46bbd48f12f6 --- /dev/null +++ b/Documentation/driver-api/ub/cdma.rst @@ -0,0 +1,190 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + +============================ +CDMA Driver Support Library +============================ + +Overview +========= +CDMA (Crystal Direct Memory Access) is used to provide asynchronous memory read +and write operations between hosts or between host and devices. + +The key features are described as follows: + ++ 1. Peer-to-peer communication between hosts, enabling bidirectional asynchronous memory read or write. ++ 2. Asynchronous memory read and write between host and devices via DMA. ++ 3. Asynchronous memory read and write between devices and host via DMA. + +This document aims to provide a guide for device driver developers on the CDMA +driver API, as well as how to use it for asynchronous memory read and write +operations between hosts in CDMA. + +CDMA Interface Operation +========================== +The API of the CDMA framework does not support arbitrary concurrent calls. +For example, using a Queue object and destroying the Queue concurrently can lead +to unexpected exceptions. +Users are required to ensure the correctness of the call logic. These objects +include context, segment, queue, etc. + +.. kernel-doc:: include/ub/cdma/cdma_api.h + :functions: + +.. kernel-doc:: drivers/ub/cdma/cdma_api.c + :export: + +CDMA API Sample +================= + +DMA Resource Sample +----------------------- +.. code-block:: c + + #define POLL_LOOP_EXAMP 100 + #define POLL_MSLEEP_EXAMP 1 + #define QUEUE_DEPTH_EXAMP 512 + #define QUEUE_RMT_EID_EXAMP 2 + #define QUEUE_DCAN_EXAMP 1 + + struct dma_seg_cfg local_seg_cfg = {}; + struct dma_seg_cfg rmt_seg_cfg = {}; + struct dma_seg *local_seg, *rmt_seg; + struct queue_cfg queue_cfg = {}; + int ctx_handle, queue_handle; + struct dma_device *dev_list; + struct dma_device *dma_dev; + u32 loop = POLL_LOOP_EXAMP; + struct dma_cr ret_cr = {}; + dma_status status; + int ret = -EINVAL; + u32 dev_num = 0; + + dev_list = dma_get_device_list(&dev_num); + if (!dev_list || !dev_num) { + printk("get device list failed\n"); + return; + } + dma_dev = &dev_list[0]; + + ctx_handle = dma_create_context(dma_dev); + if (ctx_handle < 0) { + printk("create context failed, ctx_handle: %d.\n", ctx_handle); + goto free_dev_list; + } + + queue_cfg.queue_depth = QUEUE_DEPTH_EXAMP; + queue_cfg.rmt_eid.dw0 = QUEUE_RMT_EID_EXAMP; + queue_cfg.dcna = QUEUE_DCAN_EXAMP; + queue_handle = dma_alloc_queue(dma_dev, ctx_handle, &queue_cfg); + if (queue_handle < 0) { + printk("allocate queue failed, queue_handle: %d.\n", queue_handle); + goto delete_ctx; + } + + /* Input parameter, local payload address */ + local_seg_cfg.sva = (u64)local_buf_addr; + /* Input parameter, local payload memory length */ + local_seg_cfg.len = local_buf_len; + + local_seg = dma_register_seg(dma_dev, ctx_handle, &local_seg_cfg); + if (!local_seg) { + printk("register local segment failed.\n"); + goto free_queue; + } + + /* Input parameter, remote payload address */ + rmt_seg_cfg.sva = (u64)rmt_buf_addr; + /* Input parameter, remote payload memory length */ + rmt_seg_cfg.len = rmt_buf_len; + + rmt_seg = dma_import_seg(&rmt_seg_cfg); + if (!rmt_seg) { + printk("import rmt segment failed.\n"); + goto unregister_seg; + } + + status = dma_write(dma_dev, rmt_seg, local_seg, queue_handle); + if (status != DMA_STATUS_OK) { + printk("write failed, status = %d.\n", status); + goto unimport_seg; + } + + while (loop > 0) { + ret = dma_poll_queue(dma_dev, queue_handle, 1, &ret_cr); + if (ret == 1) + break; + msleep(POLL_MSLEEP_EXAMP); + loop --; + } + ... + + unimport_seg: + dma_unimport_seg(rmt_seg); + unregister_seg: + dma_unregister_seg(dma_dev, local_seg); + free_queue: + dma_free_queue(dma_dev, queue_handle); + delete_ctx: + dma_delete_context(dma_dev, ctx_handle); + free_dev_list: + dma_free_device_list(dev_list, dev_num); + ... + +/* Register the virtual kernel online interface to notify users that + * the kernel-mode CDMA driver is online. + */ +DMA Client Sample +------------------- + +.. code-block:: c + + /* After the driver is loaded or restarted upon reset, the add + * interface is called to allow users to request resources + * required for DMA. + */ + static int example_add(u32 eid) + { + /* Refer to DMA Resource Sample, create context, queue, segment + * dma_get_device_list, dma_create_context, dma_alloc_queue etc. + */ + return 0; + } + + /* The stop interface is used to notify users to stop using the + * DMA channel. + */ + static void example_remove(u32 eid) + { + /* Refer to DMA Resource Sample, delete context, queue, segment + * dma_free_queue dma_delete_context dma_free_device_list etc. + */ + } + + /* The remove interface is used to notify users to delete resources + * under DMA. + */ + static void example_stop(u32 eid) + { + /* Stop read and write operations through status control */ + } + + static struct dma_client example_client = { + .client_name = "example", + .add = example_add, + .remove = example_remove, + .stop = example_stop, + }; + + static void example_register_client(u32 eid) + { + ... + dma_register_client(&example_client); + ... + } + +Support +======== +If there is any issue or question, please email the specific information related +to the issue or question to or vendor's support channel. diff --git a/Documentation/driver-api/ub/index.rst b/Documentation/driver-api/ub/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..0f9472ba64513b240bf119ed088a747011c4c7f9 --- /dev/null +++ b/Documentation/driver-api/ub/index.rst @@ -0,0 +1,20 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +:Copyright: |copy| 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + +============================================= +The Linux UnifiedBus implementer's API guide +============================================= + +.. class:: toc-title + + Table of contents + +.. toctree:: + :maxdepth: 4 + + ubfi + ubus + ummu-core + ubase + cdma diff --git a/Documentation/driver-api/ub/ubase.rst b/Documentation/driver-api/ub/ubase.rst new file mode 100644 index 0000000000000000000000000000000000000000..bb43d31d1949209862b238c7b6b3936dcc875f8e --- /dev/null +++ b/Documentation/driver-api/ub/ubase.rst @@ -0,0 +1,67 @@ +.. SPDX-License-Identifier: GPL-2.0+ +.. include:: + +:Copyright: |copy| 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + +UBASE Driver Support Library +----------------------------- + +.. kernel-doc:: include/ub/ubase/ubase_comm_cmd.h + :functions: + +.. kernel-doc:: include/ub/ubase/ubase_comm_ctrlq.h + :functions: + +.. kernel-doc:: include/ub/ubase/ubase_comm_debugfs.h + :functions: + +.. kernel-doc:: include/ub/ubase/ubase_comm_dev.h + :functions: + +.. kernel-doc:: include/ub/ubase/ubase_comm_eq.h + :functions: + +.. kernel-doc:: include/ub/ubase/ubase_comm_hw.h + :functions: + +.. kernel-doc:: include/ub/ubase/ubase_comm_mbx.h + :functions: + +.. kernel-doc:: include/ub/ubase/ubase_comm_qos.h + :functions: + +.. kernel-doc:: include/ub/ubase/ubase_comm_stats.h + :functions: + +.. kernel-doc:: drivers/ub/ubase/ubase_cmd.c + :export: + +.. kernel-doc:: drivers/ub/ubase/ubase_ctrlq.c + :export: + +.. kernel-doc:: drivers/ub/ubase/debugfs/ubase_debugfs.c + :export: + +.. kernel-doc:: drivers/ub/ubase/debugfs/ubase_ctx_debugfs.c + :export: + +.. kernel-doc:: drivers/ub/ubase/ubase_dev.c + :export: + +.. kernel-doc:: drivers/ub/ubase/ubase_pmem.c + :export: + +.. kernel-doc:: drivers/ub/ubase/ubase_eq.c + :export: + +.. kernel-doc:: drivers/ub/ubase/ubase_mailbox.c + :export: + +.. kernel-doc:: drivers/ub/ubase/ubase_qos_hw.c + :export: + +.. kernel-doc:: drivers/ub/ubase/ubase_stats.c + :export: + +.. kernel-doc:: drivers/ub/ubase/ubase_hw.c + :export: diff --git a/Documentation/driver-api/ub/ubfi.rst b/Documentation/driver-api/ub/ubfi.rst new file mode 100644 index 0000000000000000000000000000000000000000..a7b0466bdca487b142b4639d29a1d27f81d054f1 --- /dev/null +++ b/Documentation/driver-api/ub/ubfi.rst @@ -0,0 +1,7 @@ +.. SPDX-License-Identifier: GPL-2.0 + +UBFI Driver Support Library +--------------------------- + +.. kernel-doc:: include/ub/ubfi/ubfi.h + :functions: diff --git a/Documentation/driver-api/ub/ubus.rst b/Documentation/driver-api/ub/ubus.rst new file mode 100644 index 0000000000000000000000000000000000000000..4e39b79e9e5a301689486f07019375f51f924786 --- /dev/null +++ b/Documentation/driver-api/ub/ubus.rst @@ -0,0 +1,7 @@ +.. SPDX-License-Identifier: GPL-2.0 + +UBUS Driver Support Library +----------------------------- + +.. kernel-doc:: include/ub/ubus/ubus.h + :functions: diff --git a/Documentation/driver-api/ub/ummu-core.rst b/Documentation/driver-api/ub/ummu-core.rst new file mode 100644 index 0000000000000000000000000000000000000000..7bd07e0e1aff24f0b935f6bf3352268a99455b0d --- /dev/null +++ b/Documentation/driver-api/ub/ummu-core.rst @@ -0,0 +1,7 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +UMMU Core Support Library +--------------------------- + +.. kernel-doc:: include/linux/ummu_core.h + :functions: diff --git a/Documentation/networking/index.rst b/Documentation/networking/index.rst index 5b75c3f7a137b3dbde03903cb2a6a61c0f2facea..366e402b798d35a85bfa5664ae4f752a47ce5acb 100644 --- a/Documentation/networking/index.rst +++ b/Documentation/networking/index.rst @@ -114,6 +114,7 @@ Contents: tproxy tuntap udplite + ub/index vrf vxlan x25 diff --git a/Documentation/networking/ub/index.rst b/Documentation/networking/ub/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..a494d7c04c01e973ca1c8b4907b112b310164e73 --- /dev/null +++ b/Documentation/networking/ub/index.rst @@ -0,0 +1,15 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. include:: + +:Copyright: |copy| 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + +=============================== +The UB Networking documentation +=============================== + +.. toctree:: + :maxdepth: 1 + + UB Link Layer + UNIC Driver diff --git a/Documentation/networking/ub/ubl.rst b/Documentation/networking/ub/ubl.rst new file mode 100644 index 0000000000000000000000000000000000000000..67abc16ba052fcf19e961fd4888860b0e27c2207 --- /dev/null +++ b/Documentation/networking/ub/ubl.rst @@ -0,0 +1,69 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + +============= +UB Link Layer +============= + +Overview +======== +The ubl module implements core UB (UnifiedBus) networking functionality that +serves as the foundation for all UB networking device drivers. This module +provides essential UB link protocol handling, device setup utilities, and +standard operations that are shared across different UB networking hardware +implementations. + +UB is a new interconnection protocol that defines its own Layer 2 protocol +when integrating into the networking stack for the Linux kernel, see more +in the UB spec . + +The format of a complete UB packet is as follows: +UB Link header (UBL HDR) includes UB LINK, CC and NPI. +UB Network header consists of CC, NPI, and traditional network packet headers. + +.. code-block:: none + + <-------- UBL HDR -----------> + +--------------+------+------+---------+----------+ + | UB LINK | CC | NPI | Network | Payload | + +--------------+------+------+---------+----------+ + <------ UB Network -----> + + UB LINK: Data link layer defined by UB protocol. + CC: Congestion Control. + NPI: Network Partition Identifier. + Network: Traditional L3 header, like IPv4, IPv6 or the network control header + defined in UB. + +What the ubl module sees is as follows, as the 'cfg' field is carried through +BD (Buffer Description) for hw to construct UB LINK, the 'sw_ctype' is used +in ubl module, which corresponds to the 'cfg' field defined in UB LINK, +indicating which kind of network packet is encapsulated. + +.. kernel-doc:: include/net/ub/ubl.h + :identifiers: ublhdr + +API interface +============= +Before registering `struct net_device` to the networking stack, a UB networking +driver is supposed to allocate and set up a `struct net_device` by calling +alloc_ubldev_mqs(). + +Before passing a skb to the driver for sending, networking stack will insert the +necessary UB link layer by calling ubl_create_header() through create ops in +struct header_ops. + +Also, the driver is supposed to call ubl_type_trans() to set up the skb +correctly when it receives a packet. + +.. kernel-doc:: drivers/net/ub/dev/ubl.c + :identifiers: alloc_ubldev_mqs ubl_create_header ubl_type_trans + +An example of using the above API is the unic driver, see more detail using in +:ref:`Documentation/networking/ub/unic.rst` + +Technical Discussion +==================== +If there is any technical question about UB link layer, please start a technical +discussion by sending a mail to . diff --git a/Documentation/networking/ub/unic.rst b/Documentation/networking/ub/unic.rst new file mode 100644 index 0000000000000000000000000000000000000000..7ccba33a20f9cbe4d164eaf357cbe6d8fc82c416 --- /dev/null +++ b/Documentation/networking/ub/unic.rst @@ -0,0 +1,282 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + +=========== +UNIC Driver +=========== + +Overview +======== +unic is a UB (UnifiedBus) networking driver based on ubase driver's auxiliary +device through auxiliary bus, supporting both ethernet and UB link layer. +See :ref:`Documentation/ub/ubase/ubase.rst` for more information about ubase +driver and :ref:`Documentation/networking/ub/ubl.rst` for more information about +UB link layer. + +.. code-block:: none + + +---------------------------------------------------------------+ + | networking stack | + +---------------------------------------------------------------+ + ^ ^ ^ + | | | + | | | + v | v + +----------------+ | +---------------+ + | Ethernet Layer | | | UB Link Layer | + +----------------+ | +---------------+ + ^ | ^ + | | | + | | | + v v v + +---------------------------------------------------------------+ + | | + | unic | + | | + | +------+ +-----+ +--------+ +---------+ +-------+ +---------+ | + | | main | | dev | | netdev | | ethtool | | dcbnl | | tx & rx | | + | +------+ +-----+ +--------+ +---------+ +-------+ +---------+ | + | +---------+ +-----------+ +-----+ +---------+ +-------+ | + | | channel | | comm_addr | | crq | | rack_ip | | reset | | + | +---------+ +-----------+ +-----+ +---------+ +-------+. | + | +----+ +--------+ +-------+ +-------+ +------+ | + | | hw | | qos_hw | | event | | stats | | guid | | + | +----+ +--------+ +-------+ +-------+ +------+ | + +---------------------------------------------------------------+ + ^ ^ + | | + | | + v | + +-------------------------------+ | + | auxiliary_bus | | + +-------------------------------+ | + ^ | + | | + | | + v v + +---------------------------------------------------------------+ + | ubase | + +---------------------------------------------------------------+ + +The main submodules in unic driver: + +:: + + main + implement module_init(), module_exit() and 'struct auxiliary_driver' for + the auxiliary device of ubase driver. + + dev + implement init & uninit function and periodic task handling for unic's + netdev. + + netdev + implement 'struct net_device_ops' for unic's netdev. + + ethtool + implement 'struct ethtool_ops' for unic's netdev. + + dcbnl + implement 'dcbnl_rtnl_ops' for unic's netdev. + + tx & rx + implement packet send and receive handling. + + channel + implement channel handling for unic's netdev. + + comm_addr & rack_ip + implement the ip address handling in UB mode. + + reset + implement the entity reset handling. + + crq + implement link status change handling through ctrl (Control) receive queue. + + hw & qos_hw + implement generic hw and qos related configuration access function. + + stats + implement hw statistics collecting funciton. + + event + implement asynchronous event reporting interface. + + guid + implement the GUI (Globally Unique Identifier) querying in UB mode. + +Hardware Supported +================== + +This driver is compatible with below UB devices: + +.. code-block:: none + + +--------------+--------------+ + | Vendor ID | Device ID | + +==============+==============+ + | 0xCC08 | 0xA001 | + +--------------+--------------+ + | 0xCC08 | 0xD802 | + +--------------+--------------+ + | 0xCC08 | 0xD80B | + +--------------+--------------+ + +Note 'lsub' from ubutils package can be used to tell if the above device is +available in the system, see : + +:: + + # lsub + <00009> UB network controller <0002>: Huawei Technologies Co., Ltd. URMA management ub entity : + +Additional Features and Configurations +====================================== + +UB Link +------- +UB Link is a link layer defined by UB, which has the same layer of the existing +ethernet, and firmware will report the mode of the hardware port to the driver +through hardware capability reporting, UB_Link or ETH_MAC. + +In UB mode, the link layer is UB and its L2 address is the GUID as below +example: + +:: + + # ip -s addr + 1: ublc0d0e2: mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000 + link/ub cc:08:d8:02:d2:0a:00:00:00:00:00:00:00:00:00:01 peer 00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00 + +Note: port speed auto-negotiation is not supported in UB mode. + + +IP Address Configuration +------------------------ +IP address configuration must be performed by the management software, after +receiving the IP address configuration through crq event, the driver will +update the IP address configuration to networking stack using the netlink API. + +ELR support +----------- +ELR (Entity Level Reset) is the error recovery defined in UB, which can be +triggered by packet transmiting timeout, see unic_tx_timeout() or using the +below cmd to trigger ELR manually: + +:: + + # ethtool --reset dedicated + # echo 1 > /sys/class/net//device/reset + +Debugging +========= + +module parameters +----------------- +Enable more verbose unic driver specific debug message log by setting **debug** +to non-zero, and enable network interface debug message log by configuring +**netif_debug**, for example: + +:: + + # insmod unic.ko debug=1 netif_debug=0xFFFFF + +Debugfs Interface +----------------- +When CONFIG_DEBUG_FS is enabed, below debug info is accessible through +/sys/kernel/debug/ubase/$entity_num/unic/: + +.. code-block:: none + + ├── clear_link_status_record: clear the link status record by reading + ├── link_status_record: show the link status record debug info + ├── promisc_cfg_hw: show the promisc configuration in hardware + ├── rss_cfg_hw: show the rss configuration in hardware + ├── page_pool_info: show the rx page_pool buffer debug info + ├── caps_info: show the capability debug info + ├── dev_info: show the device info, such as max MTU + ├── qos/: show the qos related debug info + ├── vport/: show the UE (UB Entity) debug info of MUE (Management UB Entity) + ├── context/: show object context debug info, such as JFS (Jetty For Send) + ├── ip_tbl/: show the IP address configuration debug info + +Note, the bus-info in the output of below cmd can be used to query the entity +number for a unic driver's netdev, which has an entity number of "00002" as +below example: + +:: + + # ethtool -i + driver: unic + version: 1.0 + firmware-version: 1.0 + expansion-rom-version: + bus-info: 00002 + supports-statistics: yes + supports-test: yes + supports-eeprom-access: no + supports-register-dump: yes + supports-priv-flags: no + +Register Dumping +---------------- +Dump the hardware registers and report the dumpping log through vendor's support +channel using below cmd: + +:: + + # ethtool -d + +Performance tuning +================== +For different workload, the interrupt for the driver may have different cpu +pinnig policy, the below cmd can be used to set cpu pinnig policy for unic +driver's ceq (Completion Event Queue) interrupt, which is used to notify +the driver about the arrival of rx packet and completion of tx packet: + +:: + + # irq_num_list=$(cat /proc/interrupts | grep "ubase$entity_num" | grep ceq) + # echo $cpu_num > /proc/irq/$irq_num/smp_affinity_list + +CPU Intensive Workload +---------------------- +It is recommended to pin different cores to unic driver's interrupt and service +process, adjust interrupt coalesce parameters appropriately to limit interrupts +for lower CPU utilization: + +:: + + # ethtool -C rx-usecs XXX tx-usecs XXX + +Note, the **max_int_gl** in '/sys/kernel/debug/ubase/$entity_num/unic/caps_info' +is the max value of coalesce parameter. + +Latency-sensitive Workload +-------------------------- +It is recommended to pin the same core to unic driver's interrupt and service +process, disable unic driver's interrupt coalesce feature to ensure that +interrupt is triggered as soon as possible: + +:: + + # ethtool -C rx-usecs 0 tx-usecs 0 + +Manage Software +=============== +There is a manage software for UB, QOS & object context & IP address which are +related to unic driver depend on the configuration from that manage software, +refer to below debugfs for more detail on the configuration: + +:: + + QOS: /sys/kernel/debug/ubase/$entity_num/unic/qos/ + Object Context: /sys/kernel/debug/ubase/$entity_num/unic/context + IP address: /sys/kernel/debug/ubase/$entity_num/unic/ip_tbl/ip_tbl_list + +Support +======= +If there is any issue or question, please email the specific information related +to the issue or question to or vendor's support channel. diff --git a/Documentation/subsystem-apis.rst b/Documentation/subsystem-apis.rst index 90a0535a932ac6138989b96947f9227caa919d80..e2b286b797012a9c1f6780f5d79d9714ae13ea9b 100644 --- a/Documentation/subsystem-apis.rst +++ b/Documentation/subsystem-apis.rst @@ -86,3 +86,4 @@ Storage interfaces misc-devices/index peci/index wmi/index + ub/index diff --git a/Documentation/ub/cdma/cdma.rst b/Documentation/ub/cdma/cdma.rst new file mode 100644 index 0000000000000000000000000000000000000000..39be576524266d5ba92c18c17e06db8c4cb07154 --- /dev/null +++ b/Documentation/ub/cdma/cdma.rst @@ -0,0 +1,312 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + +====================================== +Crystal Direct Memory Access (CDMA) +====================================== + +Overview +========= +CDMA (Crystal Direct Memory Access) is used to provide asynchronous memory read +and write operations between hosts or between host and devices. + +The key features are described as follows: + ++ 1. Peer-to-peer communication between hosts, enabling bidirectional asynchronous memory read or write. ++ 2. Asynchronous memory read and write between host and devices via DMA. ++ 3. Asynchronous memory read and write between devices and host via DMA. + +Overall Structure +=================== + +Driver Modules +--------------- + +The CDMA driver is divided into three modules: UBASE, K-DMA, and U-DMA: + +.. code-block:: none + + +---------------------------+ + | APP | + +---------------------------+ + | + +---------------------------+ + | U-DMA | + +---------------------------+ + | | + | +-------------------+ + | | K-DMA | + | +-------------------+ + | | | + | +----------------+ | + | | Auxiliary Bus | | + | +----------------+ | + | | | + | +-------------------+ + | | UBASE | + | +-------------------+ + | | + +---------------------------+ + | CDMA Hardware | + +---------------------------+ + ++ Figure 1: CDMA Module Relationship Diagram + +UBASE provides management of hardware public resources, including CMD, mailbox +management, event management, and device reset. +It also provides a device and driver matching interface for the CDMA driver based +on the kernel auxiliary bus. + +Within the K-DMA module, functional blocks are divided according to different data +objects: Device Management is responsible for device attribute configuration +(such as EID, UPI, etc.) and device capability queries (such as Jetty specifications); +Event Management handles events reported by the controller, including completion +events and asynchronous events; +Queue Management is responsible for JFS(Jetty For Send)/JFC(Jetty For Completion) +resource management. + +Within the U-DMA module, functional blocks are divided according to data plane +functions: Memory verbs, which are unidirectional operations including read, +write, and atomic operations. +Event verbs register callback functions with K-DMA for post-processing of +asynchronous events. + +Interaction Timing +------------------- + +.. code-block:: none + + +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ + | APP | | U-DMA | | K-DMA | | UBASE | | MS | | HW | + +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ + | CDMA API | | | | | + |---------->| ioctl | | | | + | |---------->| UBASE Func| | | + | | |----------->| | | + | | |<-----------| | | + | | | HW Interface | | + | | |----------------------------------->| + | | |<-----------------------------------| + | | | UBASE Func | | | + | | |----------->| MS MSG | | + | | | |---------->| | + | | | |<----------| | + | | |<-----------| | | + | |<----------| | | | + |<----------| | | | | + | | | | | | + | CDMA API | | | | | + |---------->| HW Interface | | | + | DMA OPT |----------------------------------------------->| + | |<-----------------------------------------------| + |<----------| | | | | + | | | | | | + ++ Figure 2: CDMA Interaction Timing + +The 'Figure 2' shows the runtime sequence of interactions between the CDMA driver, +the UBASE driver, the MS(Management Software), and the hardware. + +Functionality +=============== + +CDMA device creation and reset +--------------------------------- +The CDMA devices are dynamically created by the resource management on the +management software, and the reset operation is also performed by the management +software. +Files involved: cdma_main; + +CDMA device and context management +------------------------------------ +The CDMA driver supports lifecycle management of CDMA devices and enables +applications to create device contexts based on these devices. +Files involved: cdma_context, cdma_main; + +CDMA queue management +--------------------------- +The CDMA queue includes the CDMA JFS and JFC defined on the chip, and encompasses +the management of JFS, JFC, and CTP(Compact Transport) resources. +When a remote memory read/write request is initiated, the JFS is used to fill the +corresponding WQE(Work Queue Entry), and the request execution result is received +through the JFC. +Files involved: cdma_queue, cdma_jfs, cdma_jfc, cdma_tp, cdma_db; + +CDMA segment management +----------------------------- +The CDMA driver uses local and remote segment resources for read and write operations. +These operations primarily include the register and unregister functions for +local segment resources, as well as the import and export functions for remote +segment resources. +Files involved: cdma_segment; + +CDMA read/write semantics +--------------------------- +The CDMA communication capability is implemented on the chip side as CTP mode +communication, supporting transaction operations including: +write, write with notify, read, CAS(Compare And Swap), and FAA(Fetch And Add). +Files involved: cdma_handle; + +Processing and reporting of EQE events +--------------------------------------- +The CDMA communication device supports the reporting of transaction operation +results in interrupt mode. The reported events are classified into two types: +CE(Completion Event) and AE(Asynchronous Event). +The two types of events trigger the event callback processing function registered +by the CDMA driver in advance in the interrupt context. +Files involved: cdma_event, cdma_eq; + +Supported Hardware +==================== + +CDMA driver supported hardware: + +=========== ============= +Vendor ID Device ID +=========== ============= +0xCC08 0xA003 +0xCC08 0xA004 +0xCC08 0xD804 +0xCC08 0xD805 +=========== ============= + +You can use the ``lsub`` command on your host OS to query devices. +Below is an example output: + +.. code-block:: shell + + Class <000X>: Device : + <00004> Class <0002>: Device : + +Debugging +========== + +Device Info +----------- + +.. code-block:: none + + Query CDMA device information. + Example: + $ cat /sys/kernel/debug/ubase//cdma/resource_info/dev_info + The 'CDMA_ENO' value represents the ENO (Entity Number) information for + CDMA devices. You can use the 'lsub' command on your host OS to query devices. + +Capability Info +---------------- + +.. code-block:: none + + Query CDMA device capability information. + Example: + $ cat /sys/kernel/debug/ubase//cdma/resource_info/cap_info + +Queue Info +----------- + +.. code-block:: none + + Query current queue configuration information. + Example: + $ cat /sys/kernel/debug/ubase//cdma/resource_info/queue_info + Set the queue ID value for the current query using 'queue_id' command, like + $ echo 0 > /sys/kernel/debug/ubase//cdma/resource_info/queue_id. + +Reset Info +------------ + +.. code-block:: none + + Query CDMA device reset operation records. + Example: + $ cat /sys/kernel/debug/ubase//reset_info + +JFS Context +-------------- + +.. code-block:: none + + Query the current JFS channel context information on the software side. + Example: + $ cat /sys/kernel/debug/ubase//cdma/context/jfs_context + The channel ID is configured by setting the queue ID command, like + $ echo 0 > /sys/kernel/debug/ubase//cdma/context/queue_id. + +JFS Context HW +--------------- + +.. code-block:: none + + Query the current JFS channel context information on the hardware side. + Example: + $ cat /sys/kernel/debug/ubase//cdma/context/jfs_context_hw + +JFC Context +--------------- + +.. code-block:: none + + Query the current channel JFC context information on the software side. + Example: + $ cat /sys/kernel/debug/ubase//cdma/context/sq_jfc_context + +JFC Context HW +------------------ + +.. code-block:: none + + Query the current JFC channel context information on the hardware side. + Example: + $ cat /sys/kernel/debug/ubase//cdma/context/sq_jfc_context_hw + +JFS Entity PI +------------------ + +.. code-block:: none + + Set or query the PI value of the current JFS channel, used for querying + specific SQE information of the JFS. + Example: + $ echo 0 > /sys/kernel/debug/ubase//cdma/entry_info/entry_pi + $ cat /sys/kernel/debug/ubase//cdma/entry_info/entry_pi + +JFS Entity Info +---------------- + +.. code-block:: none + + Query the information of a specific SQE for the current channel JFS. + Example: + $ cat /sys/kernel/debug/ubase//cdma/entry_info/sqe + The channel ID is configured through the queue ID command. + The SQE ID is set by configuring the 'entry_pi' as described above. + Supports kernel-space resources only. + +JFC Entity CI +---------------- + +.. code-block:: none + + Set or query the CI value of the current JFC channel, used for querying + specific CQE information of the JFC. + Example: + $ echo 0 > /sys/kernel/debug/ubase//cdma/entry_info/entry_ci + $ cat /sys/kernel/debug/ubase//cdma/entry_info/entry_ci + +JFC Entity Info +---------------- + +.. code-block:: none + + Query the information of a specific CQE for the current channel JFC. + Example: + $ cat /sys/kernel/debug/ubase//cdma/entry_info/cqe + The channel ID is configured through the Queue ID command. + The CQE ID is set by configuring the 'entry_ci' as described above. + Supports kernel-space resources only. + +Support +======== +If there is any issue or question, please email the specific information related +to the issue or question to or vendor's support channel. \ No newline at end of file diff --git a/Documentation/ub/cdma/index.rst b/Documentation/ub/cdma/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..368403170e0dd84547e23532901f41bff1bb3179 --- /dev/null +++ b/Documentation/ub/cdma/index.rst @@ -0,0 +1,14 @@ +.. SPDX-License-Identifier: GPL-2.0+ +.. include:: + +:Copyright: |copy| 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + +=============== +CDMA Driver +=============== + +.. toctree:: + :maxdepth: 2 + :numbered: + + cdma \ No newline at end of file diff --git a/Documentation/ub/index.rst b/Documentation/ub/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..22276b791363fb3099cbdf807fe0d7dbed7100a3 --- /dev/null +++ b/Documentation/ub/index.rst @@ -0,0 +1,19 @@ +.. SPDX-License-Identifier: GPL-2.0+ +.. include:: + +:Copyright: |copy| 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + +===================== +UnifiedBus Subsystem +===================== + +.. toctree:: + :maxdepth: 4 + + ub_fwctl/index + ubase/index + ubfi/index + ubus/index + ummu-core + cdma/index + urma/udma/index diff --git a/Documentation/ub/ub_fwctl/index.rst b/Documentation/ub/ub_fwctl/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..4274b33be65a4cc2daef6ebc106486621bd15a0c --- /dev/null +++ b/Documentation/ub/ub_fwctl/index.rst @@ -0,0 +1,11 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=============== +UB FWCTL Driver +=============== + +.. toctree:: + :maxdepth: 2 + :numbered: + + ub_fwctl \ No newline at end of file diff --git a/Documentation/ub/ub_fwctl/ub_fwctl.rst b/Documentation/ub/ub_fwctl/ub_fwctl.rst new file mode 100644 index 0000000000000000000000000000000000000000..5256ff8d122f094442d7633850212e50a0846fdd --- /dev/null +++ b/Documentation/ub/ub_fwctl/ub_fwctl.rst @@ -0,0 +1,112 @@ +.. SPDX-License-Identifier: GPL-2.0 + +====================== +UB FWCTL Kernel Design +====================== + +Overview +======== + +UB_FWCTL: Auxiliary bus device driver based on PMU IDEV. +It isolates user-mode debug (operation and maintenance information) functions from chip implementation details, +converts user debug commands into CMDQ commands, and sends them to the +software through the CMDQ channel of the PMU IDEV device to implement debug functions. + +Description of the Design +========================= + +The public debug tool, namely the newly added ub_fwctl tool in this document, +is primarily designed to provide functions such as querying UB public function configurations, +querying the status and statistics of many modules, and querying die-level information. + +The debug functions provided by this module are shared among multiple subsystems of UB and are not suitable +for being included in any single feature. Ub_fwctl interfaces with the open-source fwctl framework and +provides a user-defined command format for UB, supporting the public DFX functions of the UB system. + +Currently, ub_fwctl only provides querying functions and does not support configuration functions. +The DFX tools for each feature are described in detail in the corresponding feature design documents. +This design document focuses on the design of the ub_fwctl tool:: + + Purpose: As Auxiliary device driver, it provides the specific implementation of debug functions + OPS as provided by the fwctl module, and calls the CMDQ interface to pass debug messages to the software. + + Function List: + 1) Serve as an Auxiliary device driver to match Auxiliary devices. + 2) Register the fwctl device and the specific function implementation of ub_fwctl. + 3) Provide CMD queue management interfaces. + +Data structure design of UB FWCTL +================================= + +.. kernel-doc:: drivers/fwctl/ub/ub_common.h + + +System Function Design Description +================================== + +Loading and unloading the ub_fwctl driver +----------------------------------------- + +Feature Introduction:: + + FWCTL is a debug framework scheduled for integration into the mainline Linux kernel. + It provides a command pathway from userspace to kernelspace, + requiring device manufacturers to implement their + own driver plugins registered with the FWCTL kernel framework. + UB has implemented a driver called ub_fwctl, which consists of both a userspace + command-line tool (ubctl) and a kernel-space driver (ub_fwctl). After loading the ub_fwctl driver, + the sysfs system exposes a device file (such as /dev/ubcl) in the OS's /dev directory. + The userspace program ubctl can then open this device file via open(/dev/ubcl) + to obtain a file descriptor, and subsequently communicate with the driver through ioctl calls. + +Implementation Method of Function:: + + 1. Ub_fwctl registers itself with the fwctl framework. + 2. As a secondary device, ub_fwctl connects to ubase through the secondary + bus and uses the CMDQ (command queue) of The PMU IDEV to call the software + programming interface for reading and writing registers. + 3. ubctl provides command-line commands for users to invoke. + During operation, ubctl first opens the /dev/fwctl/fwctlNN device file. + It then assembles a corresponding data structure based on user input. + Next, it invokes the ioctl() system call to enter kernel mode. + Upon receiving a command from ubctl, the ub_fwctl driver first validates the command. + It then communicates with the ubase software module by calling its interface to access the CMDQ. + The software returns the register access result to ub_fwctl via the CMDQ. + ub_fwctl subsequently returns this data to user space. + Finally, after completing its operation, ubctl closes the opened /dev/ubcl file descriptor. + +.. code-block:: none + + +-------+ +----------+ +-------+ +-----+ + | ubctl | --ioctl--> | ub_fwctl | --ubase_send_cmd--> | ubase | --cmdq--> | imp | + +-------+ +----------+ +-------+ +-----+ + +Querying UB link and chip info by ub_fwctl +----------------------------------------- + +Feature Introduction:: + + After a failure occurs in the production environment, + further troubleshooting is required to identify the root cause, + including information checks such as abnormal interrupts, statistical counters, key FIFO status, + and key state machine status. The ubctl needs to support users to query the chip's + debug information through the command-line tool and output the chip's debug information + in a form that is understandable to users. + +Implementation Method of Function:: + + ubctl receives input from the command line, assembles it into corresponding commands, + and invokes ioctl to enter kernel space. The fwctl driver copies the data into the kernel space, + assembles it into the corresponding opcode, and sends the command to the software for processing via + the CMDQ of the PMU IDEV. After reading the corresponding registers according to the chip's rules, + the software returns the data to ub_fwctl, which then returns the data to user space. + Finally, ubctl displays the data. + + The following types of registers are supported for query: + 1. Querying information about the UB link. + 2. Querying QoS memory access information. + 3. Querying port link status. + 4. Querying DL layer service packet statistics. + 5. Querying NL layer service packet statistics. + 6. Querying SSU packet statistics. + 7. Querying BA layer packet statistics. diff --git a/Documentation/ub/ubase/index.rst b/Documentation/ub/ubase/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..5fcc9347e1e96d2d02c9481266ebfbf930a597ab --- /dev/null +++ b/Documentation/ub/ubase/index.rst @@ -0,0 +1,14 @@ +.. SPDX-License-Identifier: GPL-2.0+ +.. include:: + +:Copyright: |copy| 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + +============= +UBASE Driver +============= + +.. toctree:: + :maxdepth: 2 + :numbered: + + ubase \ No newline at end of file diff --git a/Documentation/ub/ubase/ubase.rst b/Documentation/ub/ubase/ubase.rst new file mode 100644 index 0000000000000000000000000000000000000000..b4cb90820a53c715d891a2d19b682a2764a41873 --- /dev/null +++ b/Documentation/ub/ubase/ubase.rst @@ -0,0 +1,259 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + +============================== +UNIFIEDBUS BASE DRIVER (UBASE) +============================== + +UB is a new interconnection protocol and architecture designed for computing +systems, see more on the UB spec . + +This document will introduce the composition of the UBASE Driver and how to +write a UB auxiliary device driver under the Auxiliary Bus framework of UBASE, +also include related debugging information. + +Overview +======== + +UBASE driver is one of the base driver for UB network, based on UB hardware +interface, providing public resource management and abstraction of common +interfaces for the upper layer UB device drivers such as unic, udma, cdma, fwctl, +pmu and uvb, which are auxiliary devcie drivers. UBASE driver also offers +device-driver matching interfaces for the upper-layer drivers based on auxiliary +bus, isolating different auxiliary device drivers, like udma driver using urma +core and unic driver with TCP/IP stack. And ubase has the ability of extending +auxiliary device list to transfer the UB hardware for further use and richer +features. + +UBASE includes the functionalities of ubus device management, resource management, +auxiliary device management, entity management, query the specific capabilities +of the device and so on. It's also the base of other auxiliary device drivers +which must be loaded before them. + +.. code-block:: none + + +----------+ +----------+ +----------+ +---------+ +-----------+ +---------+ + | unic | | udma | | cdma | | pmu | | ubctl | | uvb | + +----------+ +----------+ +----------+ +---------+ +-----------+ +---------+ + ^ ^ ^ ^ ^ ^ + | | | | | | + v v v v v v + +--------------------------------------------------------------------------+ + | auxiliary bus | + +--------------------------------------------------------------------------+ + ^ + | + v + +--------------------------------------------------------------------------+ + | ubase | + | +-----+ +------+ +---------+ +-------+ +----+ +-----+ +-----+ +------+ | + | | dev | | main | | debugfs | | ctrlq | | eq | | arq | | ras | | ubus | | + | +-----+ +------+ +---------+ +-------+ +----+ +-----+ +-----+ +------+ | + | +-----+ +----+ +---------+ +------+ +-----+ +-------+ +----+ +-------+ | + | | cmd | | hw | | mailbox | | pmem | | qos | | reset | | tp | | stats | | + | +-----+ +----+ +---------+ +------+ +-----+ +-------+ +----+ +-------+ | + +--------------------------------------------------------------------------+ + ^ ^ ^ + | | | + | v v + | +------------------+ +------------------+ + | | ubus | | ummu | + | +------------------+ +------------------+ + | ^ ^ + | | | + v v v + +--------------------------------------------------------------------------+ + | firmware | + +--------------------------------------------------------------------------+ + +Below is the summary for the submodules in ubase driver: + + - 1) main: implement module_init/exit(). + - 2) dev: implement auxiliary bus init/uninit function, resource creating and + auxiliary device enable/disable. + - 3) cmd: implement 'command queue' to interact with firmware. + - 4) ctrlq: implement 'control queue' to interact with management software. + - 5) mailbox: implement 'mailbox' configuration to interact with hardware + through `cmdq`. + - 6) hw: implement interaction with firmware and hardware for functions. + - 7) reset: implement hardware reset handling for ubase driver. + - 8) tp: implement tp layer context BA and context creation. + - 9) debugfs: implement kernel debugfs to obtain debugging information. + - 10) qos: implement quality of service for upper communication modules. + - 11) ras: implement hardware error handler. + - 12) ubus: implement interaction with module `ubus`. + - 13) eq: event queue including asynchronous and completion event. + +Supported Hardware +================== + +UBUS vendor/device pairs: + +========= =========== ====================================== +Vendor ID Device ID Description +========= =========== ====================================== +0xCC08 0xA001 Kunpeng URMA MUE (Management UB Entity) +0xCC08 0xA002 Kunpeng URMA UE (UB Entity) +0xCC08 0xA003 Kunpeng CDMA MUE (Management UB Entity) +0xCC08 0xA004 Kunpeng CDMA UE (UB Entity) +0xCC08 0xA005 Kunpeng PMU MUE (Management UB Entity) +0xCC08 0xA006 Kunpeng PMU UE (UB Entity) +0xCC08 0xD802 Ascend URMA MUE (Management UB Entity) +0xCC08 0xD803 Ascend URMA UE (UB Entity) +0xCC08 0xD804 Ascend CDMA MUE (Management UB Entity) +0xCC08 0xD805 Ascend CDMA UE (UB Entity) +0xCC08 0xD806 Ascend PMU MUE (Management UB Entity) +0xCC08 0xD807 Ascend PMU UE (UB Entity) +0xCC08 0xD80B Ascend UBOE MUE (Management UB Entity) +0xCC08 0xD80C Ascend UBOE UE (UB Entity) +========= =========== ====================================== + +Supported Auxiliary device +========================== + +UB Auxiliary bus device/driver pairs: + +========= ==== ==== ==== ===== === === +Device ID unic udma cdma fwctl pmu uvb +========= ==== ==== ==== ===== === === +0xA001 O O X X X O +0xA002 X O X X X O +0xA003 X X O X X X +0xA004 X X O X X X +0xA005 X X X O O X +0xA006 X X X O O X +0xD802 O O X X X O +0xD803 X O X X X O +0xD804 X X O X X X +0xD805 X X O X X X +0xD806 X X X O O X +0xD807 X X X O O X +0xD80B O O X X X O +0xD80C X O X X X O +========= ==== ==== ==== ===== === === + +If anyone wants to support a new auxiliary device driver based on ubase, after +adding an specific device id matched with vendor id, extending the driver +list is necessary as follows:: + enum ubase_drv_type { + UBASE_DRV_UNIC, + UBASE_DRV_UDMA, + UBASE_DRV_CDMA, + UBASE_DRV_FWCTL, + UBASE_DRV_PMU, + UBASE_DRV_UVB, + UBASE_DRV_MAX, + }; + +Next, `struct ubase_adev_device` is supposed to be extended by the new device +driver with its name filled in `suffix` and supported capabilities function +hooking up to the handling named `is_supported`. Following is an example driver +`unic` in ``ubase_dev.c``:: + static struct ubase_adev_device { + const char *suffix; + bool (*is_supported)(struct ubase_dev *dev); + } ubase_adev_devices[UBASE_DRV_MAX] = { + [UBASE_DRV_UNIC] = { + .suffix = "unic", + .is_supported = &ubase_dev_unic_supported + }, + }; + +Then the new driver can fulfill `struct auxiliary_driver` ops allowing auxiliary +bus transfer handling `probe` to initialize the new driver and handling `remove` +to uninitialize it. + +Module parameters +================= +UBASE driver includes one module parameter for now as `debug`. The default +parameter can support full function of ubase and related drivers, but in some +special scene like locating problems, debug information is necessary. + +debug + +This parameter controls the print level of ubase driver, preventing printing +debug information like `UBUS ELR start` to locate the position of driver running, +which may be helpful when doing problem identification to clarify the line of +code where the problem occurs. + +This parameter is not supposed to set in loading driver but changed in a system +configuration file created by ubase, which set to be `0` means disable in default +as not showing all debug information. If the user wants to enable the debug printing, +the file `/sys/module/ubase/parameters/debug` can be set to an integer value +`except 0` as enable like `1` through the command `echo`, following line shows:: + echo 1 > /sys/module/ubase/parameters/debug + +Or set the insmod parameter `debug` to value `1`, like the following: + +.. code-block:: none + + insmod ubase.ko debug=1 + +Debugging +========= +UBASE driver supports to obtain debug related information for users through +`debug filesystem` set by Linux kernel, which helps a lot for problem locating +and quick overview of ubase driver. The ubase debugfs includes +`context querying in software and hardware`, `reset information`, +`capabilities information`, `activate record`, `qos information`, +`prealloc memory information`. + +Through debugfs interfaces when `CONFIG_DEBUG_FS` is enabled, the users can obtain +these information from the directory in system:: + /sys/kernel/debug/ubase// + +1) context querying + + UBASE driver supports to query the context created in ubase for all auxiliary + device drivers, including aeq, ceq, tp and tpg context stored in both software + and hardware to verify whether configuration satisfy using demand. Note, the + context ended with `hw` means hardware, like `aeq_context_hw`, and another one + without `hw` means that stored in software, such as `aeq_context`. + +2) reset information + + UBASE driver supports to query all kinds of reset implementation counts, + including ELR reset, port reset, himac reset, total finish count in software + and hardware, and failed count. + +3) capabilities information + + UBASE driver supports to query the capabilities information in self device, + which can be used for upper-layer drivers, and also resource size for creating. + +4) activate record + + UBASE driver supports to query activate record about the hardware, including + activate and deactivate counts and the exact time of these actions. + +5) qos information + + UBASE driver supports to query quality of service(qos) information configured + in hardware, about configuration set by ets, tm and `management software`, + including `tc &tc group`, `rqmt table`, `mapping in vl, sl and dscp`, + `tm port`, `tm priority`, `qset and queue information` and so on. + +6) prealloc memory information + + UBASE driver supports to query the hulk pages allocated by ubase for both + common use and udma use. + +Functionality dependencies +========================== +Some functions in ubase driver rely on configuration of `management software` as +the manager, the following shows dependencies of ubase: + - 1) Qos configuration: `management software` take the responsibility of + `entity creation and distribute TQMS queue` and `mapping from sl to vl`. + - 2) TP context: `management software` take the response of creating TP layer + context for common use, including tp context basic address (BA), + tp group(tpg) context BA, TP extdb buff, TP timer buff, CC context BA, + Dest address BA, Seid_upi BA, TPM BA and so on. + - 3) Reset: The reset process needs collaborative cooperation between ubase + driver and `management software`, including stop flow, resume flow, + reconstruct TP layer context and so on. + +Support +======= +If there is any issue or question, please email the specific information related +to the issue or question to or vendor's support channel. \ No newline at end of file diff --git a/Documentation/ub/ubfi/index.rst b/Documentation/ub/ubfi/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..2dd11600f4f747abea30417efd1dea828a096f17 --- /dev/null +++ b/Documentation/ub/ubfi/index.rst @@ -0,0 +1,11 @@ +.. SPDX-License-Identifier: GPL-2.0 + +======================= +UB Firmware Spec Driver +======================= + +.. toctree:: + :maxdepth: 2 + :numbered: + + ubfi diff --git a/Documentation/ub/ubfi/ubfi.rst b/Documentation/ub/ubfi/ubfi.rst new file mode 100644 index 0000000000000000000000000000000000000000..efea335726b81fbeb1b1ba5d7b119545b81c0ca1 --- /dev/null +++ b/Documentation/ub/ubfi/ubfi.rst @@ -0,0 +1,178 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=========== +UBFI Driver +=========== + +What is UBFI +============ + +When BIOS boots the OS with UB firmware, it should report the UB-related +information in the system so that the OS can obtain the UB-related information, +including UBC, UMMU, and all other information required for UB enabling. + +Startup information is related to chip specifications and is static information +that can be reported through a static information table. There are three +reporting methods: UBIOS, ACPI, and Device Tree. The only difference among these +three methods lies in the entry points for obtaining the UB-related information +tables. The contents of each information table remain consistent. + +UnifiedBus Firmware Interface (UBFI) driver supports obtaining UB-related +information from the BIOS via the ACPI table or device tree. And create software +instances of UBCs and UMMUs in the OS. + +UBFI driver is one of the fundamental drivers of UB. It has achieved the +aforementioned functions. + +.. code-block:: none + + +--------------------------------------------------------------+ + | BIOS | + +--------------------------------------------------------------+ + ^ ^ + |acpi of| + v v + +--------------------------------------------------------------+ + | kernel | + +--------------------------------------------------------------+ + ^ + | + v + +--------------------------------------------------------------+ + | ubfi | + +--------------------------------------------------------------+ + ^ ^ + | | + v v + +-----------------+ +-----------------+ + | ubus | | ummu | + +-----------------+ +-----------------+ + +What does UBFI do +================= + +When loading the ubfi driver, it detects the current OS boot mode and retrieves +the UBRT (UB root table) physical address from the BIOS. + + - ACPI (UBRT table) + - device tree (node: chosen: ubios-information-table) + +For the structure of UBRT, please refer to https://www.unifiedbus.com/ + +Create UBC +---------- + +BIOS may report information about multiple UBCs, some of which is shared among +multiple UBCs and is reported in ``struct ubrt_ubc_table`` + +.. kernel-doc:: drivers/ub/ubfi/ubc.h + :functions: ubrt_ubc_table + +As ``ubc_cna_start``, ``ubc_cna_end``, ``ubc_eid_start``, ``ubc_eid_end``, +``ubc_feature``, ``cluster_mode``, these attributes belong to the entire UBPU +node and are shared by all UBCs. + +For a single UBC, its information is reported in the ``struct ubc_node`` + +.. kernel-doc:: drivers/ub/ubfi/ubc.h + :functions: ubc_node + +We have performed the following work on a single UBC. + + - Create the UBC structure and record the UBC information + - Register the UBC irq with the kernel + - Initialize UBC and register the UBC device with the kernel + - Register the MMIO address space of UBC with the kernel + - Set the MSI domain for all UBCs + +After completing these steps, ``struct list_head ubc_list`` will be provided +externally, which records all UBCs within the node for subsequent +interconnection and communication purposes. + +Set MSI domain for UBC +~~~~~~~~~~~~~~~~~~~~~~ + +UBFI driver requests interrupts from the interrupt management subsystem on +behalf of the entity and delivers the interrupt configuration to the entity. +When reporting an interrupt, the entity writes the interrupt information into +the interrupt controller, which then calls back the interrupt management +subsystem. The interrupt management subsystem subsequently invokes the UB driver +to handle the corresponding interrupt. + +UB created a new Message Signaled Interrupt domain called USI (UB Signaled +Interrupt). + +UB will add a platform device in the DSDT and IORT tables to associate UBC +with the USI domain. If booting with device tree, we will add a new UBC node in +DTS for binding the USI domain. For each UBC, a corresponding number of platform +devices should be created. We will set the USI domain of these platform devices +to the USI domain of each UBC. + +Example in DTS for UBC:: + + ubc@N { + compatible = "ub,ubc"; + #interrupt-cells = <0x3>; + interrupt-parent = <0x01>; + interrupts = <0x0 0xa 0x4>; + index = <0x00>; + msi-parent = <0x1 0xabcd>; + }; + +Parse UMMU and PMU +------------------ + +Both UMMU and UMMU-PMU devices are platform devices and support creation via +ACPI and DTS. + +ACPI method: + - The device information for UMMU and UMMU-PMU has been added to DSDT and + IORT tables. + - When the OS enables ACPI functionality, the ACPI system will recognize + the device information in the DSDT and IORT tables and automatically + create platform devices for UMMU and UMMU-PMU. + - The number of platform devices for UMMU and UMMU-PMU depends on the + number of device information nodes described in the DSDT and IORT tables. + +DTS method: + - The DTB file has added device tree nodes for UMMU and UMMU-PMU. + - When the OS enables the device tree functionality, the DTS system will + recognize the device tree nodes for UMMU and UMMU-PMU, and then + automatically create platform devices for them. + - The number of platform devices for UMMU and UMMU-PMU depends on the + number of corresponding device tree nodes described in the device tree. + + Example in DTS for UMMU and UMMU-PMU:: + + ummu@N { + compatible = "ub,ummu"; + index = <0x0>; + msi-parent = <&its>; + }; + + ummu-pmu@N { + compatible = "ub,ummu_pmu"; + index = <0x0>; + msi-parent = <&its>; + }; + +Obtain UMMU nodes from the UBRT table: + - The UBRT table can be parsed to extract the UMMU sub-table, which contains + several UMMU nodes. Each UMMU node describes the hardware information of an + UMMU device and its corresponding UMMU-PMU device. The specific content of + UMMU nodes can be found in ``struct ummu_node``. + + - The number of UMMU platform devices created via ACPI or DTS should match the + number of UMMU nodes in the UBRT table, as they have a one-to-one + correspondence. The same one-to-one correspondence applies to UMMU-PMU + devices and UMMU nodes. + +Configure UMMU and PMU devices: + - For each UMMU node parsed from the UBRT table, the register information and + NUMA affinity described in the UMMU node can be configured for the + corresponding UMMU and UMMU-PMU devices. + - Each UMMU node's content is stored in the ``ubrt_fwnode_list`` linked list. + Subsequently, the corresponding UMMU node can be found by using the fwnode + property of the UMMU and UMMU-PMU devices, making it convenient to obtain the + hardware information during the initialization of the UMMU and UMMU-PMU + drivers. \ No newline at end of file diff --git a/Documentation/ub/ubus/hisi_ubus.rst b/Documentation/ub/ubus/hisi_ubus.rst new file mode 100644 index 0000000000000000000000000000000000000000..b384b058129f53498af8e0da8113d553fae209b4 --- /dev/null +++ b/Documentation/ub/ubus/hisi_ubus.rst @@ -0,0 +1,95 @@ +.. SPDX-License-Identifier: GPL-2.0 + +===================== +Hisilicon UBUS Driver +===================== + +Hisilicon UBUS Driver (abbreviated as Hisi UBUS) is a UnifiedBus (UB) +specification management subsystem specifically implemented for Hisi chips. It +provides a subsystem operation interfaces implementation:: + + static const struct ub_manage_subsystem_ops hisi_ub_manage_subsystem_ops = { + .vendor = HISI_VENDOR_ID, + .controller_probe = ub_bus_controller_probe, + .controller_remove = ub_bus_controller_remove, + .ras_handler_probe = ub_ras_handler_probe, + .ras_handler_remove = ub_ras_handler_remove + }; + +including probe/remove methods for the UB bus controller and ub ras handler. +Each specification management subsystem has a unique vendor id to identify the +provider. This vendor id is set to the vendor field of +``ub_manage_subsystem_ops`` implementation. During UB bus controller probe, a +ub_bus_controller_ops will be set to the UB bus controller, message device and +debug file system will be initialized. During UB bus controller remove, ops +will be unset, message device will be removed and debug file system will be +uninitialized. + +During module init, hisi_ub_manage_subsystem_ops is registered to Ubus driver +via the ``register_ub_manage_subsystem_ops()`` method provided by Ubus driver:: + + int register_ub_manage_subsystem_ops(const struct ub_manage_subsystem_ops *ops) + +When module is being unloaded, Ubus driver's +``unregister_ub_manage_subsystem_ops()`` is called to unregister the subsystem +operation interfaces:: + + void unregister_ub_manage_subsystem_ops(const struct ub_manage_subsystem_ops *ops) + +Hisi UBUS Controller Driver +=========================== +Hisi UBUS provides a ub bus controller operation interfaces implementation:: + + static struct ub_bus_controller_ops hi_ubc_ops = { + .eu_table_init = hi_eu_table_init, + .eu_table_uninit = hi_eu_table_uninit, + .eu_cfg = hi_eu_cfg, + .mem_decoder_create = hi_mem_decoder_create, + .mem_decoder_remove = hi_mem_decoder_remove, + .register_ubmem_irq = hi_register_ubmem_irq, + .unregister_ubmem_irq = hi_unregister_ubmem_irq, + .register_decoder_base_addr = hi_register_decoder_base_addr, + .entity_enable = hi_send_entity_enable_msg, + }; + +including init/uninit method for EID/UPI table, create/remove method for UB +memory decoder, register/unregister method for UB memory decoder interrupts +and so on. + +UB Message Core Driver +====================== +Hisi UBUS implements a message device that provides a set of operations:: + + static struct message_ops hi_message_ops = { + .probe_dev = hi_message_probe_dev, + .remove_dev = hi_message_remove_dev, + .sync_request = hi_message_sync_request, + .response = hi_message_response, + .sync_enum = hi_message_sync_enum, + .vdm_rx_handler = hi_vdm_rx_msg_handler, + .send = hi_message_send, + }; + +including synchronous message sending, synchronous enumeration message +sending, response message sending, vendor-defined message reception handling +and so on. After device creation, ``message_device_register()`` method of Ubus +driver is called to register the device to the Ubus driver message framework:: + + int message_device_register(struct message_device *mdev) + +This framework provides a unified interface for message transmission and +reception externally. + +Hisi UBUS Local Ras Error Handler +================================= +Hisi UBUS provides a local RAS handling module to detect and process errors +reported on the UB bus. It offers error printing and registry dump, determines +whether recovery is needed based on error type and severity, and can reset +ports for port issues in cluster environment. + +UB Vendor-Defined Messages Manager +================================== +Hisi UBUS defines several vendor-defined messages, implements messages' +transmission and processing. These private messages are mainly used for +managing the registration, release, and state control of physical and +virtual devices. \ No newline at end of file diff --git a/Documentation/ub/ubus/index.rst b/Documentation/ub/ubus/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..a4c2a58324cffef337bb2a98579ff818b3856d2e --- /dev/null +++ b/Documentation/ub/ubus/index.rst @@ -0,0 +1,13 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============= +UB BUS Driver +============= + +.. toctree:: + :maxdepth: 2 + :numbered: + + ubus + ubus-service + hisi_ubus \ No newline at end of file diff --git a/Documentation/ub/ubus/ubus-service.rst b/Documentation/ub/ubus/ubus-service.rst new file mode 100644 index 0000000000000000000000000000000000000000..fd347fff959a369c22aeab3bbd018fb2ad895f79 --- /dev/null +++ b/Documentation/ub/ubus/ubus-service.rst @@ -0,0 +1,60 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=========================================== +UnifiedBus Bus Driver (UBUS Driver) Service +=========================================== + +The UnifiedBus (UB) specification describes RAS-related error handling and +notification-based hot-plug functionalities. The UBUS driver implements these +two types of functionalities as two independent services in software. This +article will separately introduce these two services. + +UB Device Driver Error Service +============================== +The UB specification defines three categories of protocol errors: A, B, and C. +Among these, A and B category protocol errors are directly handled by the +UB device driver, and thus will not be further discussed in this document. +C category protocol errors are reported to the UBUS Driver via the APEI +mechanism. The UBUS Driver provides a set of mechanisms for handling C category +protocol errors, which work in conjunction with the UB device driver to +complete the error handling process. + +The UBUS driver provides the ``struct ub_error_handlers`` structure, which +includes multiple callback functions related to error handling. The UB device +driver needs to implement these callback functions:: + + struct ub_error_handlers { + void (*ub_reset_prepare)(struct ub_entity *uent); + void (*ub_reset_done)(struct ub_entity *uent); + ub_ers_result_t (*ub_error_detected)(struct ub_entity *uent, ub_channel_state_t state); + ub_ers_result_t (*ub_resource_enabled)(struct ub_entity *uent); + }; + +For UB device driver: + + - ub_reset_prepare is called before ELR, serving to notify the device driver to + prepare for the work before ELR + - ub_reset_done is called after ELR, serving to notify the device driver that + ELR has completed and services can be resumed + - ub_error_detected is called when the UB bus driver detects an error, serving + to notify the UB device driver of the occurrence of an error + - ub_resource_enabled is called after the UB bus driver has completed error + handling, serving to notify the UB device driver that error handling has + completed + +Hot-Plug Service +================ +The UB specification defines the hot-plug functionality for devices, which +requires coordination between software and hardware. The UBUS driver implements +the hot removal and hot insertion of external devices on a per-slot basis. +For detailed procedures, please refer to the UB specification document. The main +functional points implemented by the UBUS driver include: + + - Button event handling, completing the processing of hot-plug and + hot-unplug button messages + - Indicator control, switching different knowledge points based on the + device status + - Power control, performing power on/off operations for slots based on the + device status + - Providing a user-space sysfs interface to simulate button effects + according to user commands diff --git a/Documentation/ub/ubus/ubus.rst b/Documentation/ub/ubus/ubus.rst new file mode 100644 index 0000000000000000000000000000000000000000..e7176e98732e626cdbf43fdee5e0ca17e0c66853 --- /dev/null +++ b/Documentation/ub/ubus/ubus.rst @@ -0,0 +1,312 @@ +.. SPDX-License-Identifier: GPL-2.0 + +====================================== +How To Write Linux UB Device Drivers +====================================== + +UnifiedBus (abbreviated as UB) is an interconnection technology and +architecture designed for computing systems. It unifies the communication +between IO, memory access, and various processing units within the same +interconnection technology framework, enabling high-performance data transfer, +unified resource management, efficient collaboration, and effective programming +in computing systems. Resource management is one of its key features, +implemented through a combination of software and hardware. The UB Bus Driver +(referred to as the UBUS Driver) implements the software portion of this +feature. This document provides a brief overview of the components within the +UBUS Driver framework and how to develop UB device drivers within this driver +framework. See more on the UB spec . + +Composition of the UBUS Driver +============================== +The UBUS Driver consists of two parts. The first part is the common +implementation section, which will be developed according to the UB +specification requirements. The second part is the proprietary implementation by +each manufacturer, which is based on the specific circuit designs of each host +manufacturer. Each host manufacturer can provide differentiated functionalities +in this part of the code. + +If the UBUS subsystem is not configured (CONFIG_UB_UBUS is not set), most of +the UBUS functions described below are defined as inline functions either +completely empty or just returning an appropriate error codes to avoid +lots of ifdefs in the drivers. + +The figure below illustrates the internal composition and system boundaries of +the UBUS Driver. + +.. code-block:: none + + +----------------------------------------------------------+ + | ub device driver | + +----------------------------------------------------------+ + ^ + | + v + +----------------------------------------------------------+ + | ubus driver | + | | + | +--------------------------------------------------+ | + | | ubus driver vendor-specific | | + | +--------------------------------------------------+ | + | | + | +--------------------------------------------------+ | + | | ubus driver common | | + | | | | + | | +------+ +--------+ +------+ +-------+ +-----+ | | +---------+ + | | | enum | | config | | port | | route | | msg | | | <-> | GIC/ITS | + | | +------+ +--------+ +------+ +-------+ +-----+ | | +---------+ + | | +------------+ +--------+ +-----------+ | | + | | | controller | | entity | | interrupt | | | +------------+ + | | +------------+ +--------+ +-----------+ | | <-> | IOMMU/UMMU | + | | +---------+ +----------+ +------+ +----------+ | | +------------+ + | | | decoder | | resource | | pool | | instance | | | + | | +---------+ +----------+ +------+ +----------+ | | + | | +-----+ +------+ +-------+ +---------+ +-------+ | | + | | | ras | | link | | reset | | hotplug | | sysfs | | | + | | +-----+ +------+ +-------+ +---------+ +-------+ | | + | | +------+ +---------------+ | | + | | | ubfi | | bus framework | | | + | | +------+ +---------------+ | | + | +--------------------------------------------------+ | + +----------------------------------------------------------+ + ^ + | + v + +----------------------------------------------------------+ + | hardware/firmware | + +----------------------------------------------------------+ + +The following briefly describes the functions of each submodule within the +UBUS driver: + + - enum: implement network topology scanning and device enumeration + functionality + - config: enable access to the device configuration space + - port: manage device ports + - route: implement the configuration of the routing table + - msg: implement message assembly and transmission/reception processing + for management messages + - controller: initialization and de-initialization of the UB controller + - entity: enable device configuration, multi-entity management, and other + functionalities + - interrupt: implement USI interrupt functionality + - decoder: implement address decoding functionality for MMIO access to + device resource space + - resource: manage the MMIO address space allocated by the user host to + the device + - pool: implementation of pooled message processing + - instance: implement bus instance management + - ras: implement handling for RAS exceptions + - link: implement processing of link messages + - reset: implement the reset function + - hotplug: enable hot-plug functionality for the device + - sysfs: implement sysfs attribute files + - ubfi: implement parsing of the UBRT table + - bus framework: implementation of the Ubus Driver Framework + +Structure of UB device driver +============================= +In Linux, the ``ub_driver`` structure is used to describe a UB device driver. +The `struct ub_driver` is employed to represent a UB device driver, and +the structure definition is as follows. + +.. kernel-doc:: include/ub/ubus/ubus.h + :functions: ub_driver + +This structure includes a matchable device table (`id_table`), a probe function, +a remove function, a shutdown function, error handling, and other functionalities. +The following content will provide a reference for the implementation of these +features. + +Rules for Device and Driver Matching +------------------------------------ +The matching rules for UnifiedBus devices and drivers are relatively flexible, +allowing for any combination of the following five matching entries in the +`struct ub_device_id` within the device driver to achieve the target matching rule: + + - GUID's Vendor ID + - GUID's Device ID + - Configuration Space Module Vendor ID + - Configuration Space Module ID + - Configuration Space Class Code + +The ID table is an array of ``struct ub_device_id`` entries ending with an +all-zero entry. Definitions with static const are generally preferred. + +.. kernel-doc:: include/linux/mod_devicetable.h + :functions: ub_device_id + +Most drivers only need ``UB_ENTITY()`` or ``UB_ENTITY_MODULE`` or +``UB_ENTITY_CLASS()`` to set up a ub_device_id table. + +The following is an example:: + + static const struct ub_device_id sample_tbl[] = { + { 0xCC08, 0xA001, UB_ANY_ID, UB_ANY_ID, 0, 0 }, + { UB_ENTITY(0xCC08, 0xA001), 0, 0 }, + { UB_ENTITY_MODULE(0xCC08, 0xA001, 0xCC08, 0xA001), 0, 0 }, + { UB_ENTITY_CLASS(0x0200, 0xffff) }, + }; + +New UB IDs may be added to a device driver ub_ids table at runtime +as shown below:: + + echo "vendor device modulevendor moduleid class class_mask driver_data" > \ + /sys/bus/ub/drivers/sample/new_id + +All fields are passed in as hexadecimal values (no leading 0x). +The vendor and device fields are mandatory, the others are optional. Users +need pass only as many optional fields as necessary: + + - modulevendor and moduledevice fields default to UB_ANY_ID (FFFFFFFF) + - class and classmask fields default to 0 + - driver_data defaults to 0UL. + - override_only field defaults to 0. + +Note that driver_data must match the value used by any of the ub_device_id +entries defined in the driver. This makes the driver_data field mandatory +if all the ub_device_id entries have a non-zero driver_data value. + +Once added, the driver probe routine will be invoked for any unclaimed +UB devices listed in its (newly updated) ub_ids list. + +Register UB Device Driver +------------------------- +The UB device driver uses `ub_register_driver` to register the device driver. +During the registration process, the matching between the device and the +driver will be triggered, with the matching rules referenced in the previous +section. + +UB Device Driver Probe Process Reference +---------------------------------------- +- Call `ub_set_user_info` to configure the user host information into the entity + Each entity's configuration space has corresponding user register + information, such as user EID, token ID, etc. Before the device driver + starts using the device, it needs to configure the user host information + for the device. + +- Call `ub_entity_enable` to configure the access path between the host and the device + Before using the device, you need to enable the bidirectional channel + switch for accessing the device from the user host and vice versa. + This is achieved by configuring the device configuration space registers. + +- Set the DMA mask size + The device driver can reconfigure this field segment based on the + device's DMA addressing capability. The default configuration is 32-bit. + +- Call the kernel DMA interface to request DMA memory + The device driver requests DMA memory through the DMA interface provided + by the kernel to prepare for subsequent device DMA operations. + +- Call `ub_iomap` to complete the MMIO access mapping for the resource space + The device resource space stores private configurations related to device + driver capabilities. Before accessing the device resource space, you need + to call the ioremap interface to complete address mapping. The ub_iomap + interface uses the device attribute, while the ub_iomap_wc interface + uses the writecombine attribute. + +- Call `ub_alloc_irq_vectors` or `ub_alloc_irq_vectors_affinity` to complete + the interrupt request, and then call the kernel's interrupt registration API. + +- Initiate specific business functions + +UB Device Driver Removal Process Reference +------------------------------------------ +- Stop specific business functions +- Invoke the kernel's interrupt unregistration API, call ub_disable_intr, to + complete the unregistration of the interrupt handler and release the interrupt +- Call ub_iounmap to demap the MMIO access space +- Invoke the kernel's DMA interface to release DMA memory +- Call ub_entity_enable to close the access path between the host and the device +- Call ub_unset_user_info to clear the user host information configured to the + entity + +UB Device Driver Shutdown +------------------------- +The UB device shutdown is triggered during the system shutdown or restart +process, and the UB device driver needs to stop the service flow in the shutdown +interface. + +UB Device Driver Virtual configure +---------------------------------- + +If the MUE supports multiple UEs, the device driver needs to provide +`virt_configure` callback. the UEs can be enabled or disabled to facilitate +direct connection to virtual machines for use. The bus driver will cyclically +call the virt_configure callback of the device driver to enable and disable +each UE in sequence. Within the virt_configure function of the device driver, +it needs to call `ub_enable_ue` and `ub_disable_ue` provided by the bus driver +to create and destroy UEs, at the same time, private processing logic can +also be executed. + +UE can be enabled and disabled through sysfs. The process is as follows:: + + 1. Check the number of UEs currently supported by the MUE + # cat /sys/bus/ub/devices/.../ub_totalues + 2. Specify the number of enabled UEs within the maximum UE quantity range + # echo 3 > /sys/bus/ub/devices/.../ub_numues + 3. Disable UEs + # echo 0 > /sys/bus/ub/devices/.../ub_numues + +UB Device Driver Virtual notify +------------------------------- + +If the device supports multiple UEs and the MUE device driver wants to be +aware of UE state changes, `virt_notify` hook function can be implemented to +capture the UE state. + +UB Device Driver Activate and Deactivate +---------------------------------------- + +The bus driver supports maintaining the working status of entities, indicating +whether an entity is in operation. It also provides corresponding interfaces +for controlling devices to enter or exit the working state, such as +`ub_activate_entity` and `ub_deactivate_entity`. If the device driver needs +to perform any special procedures, it must implement the corresponding activate +and deactivate hook functions. + +UB Device Driver RAS handler +---------------------------- + +The bus driver provides a set of hooks for RAS processing, creating an +opportunity window to notify device drivers when handling events such as +resets and RAS, allowing them to execute corresponding processing measures. +Currently implemented hooks include `reset_prepare`, `reset_done`, +`error_detected`, and `resource_enabled`. Device drivers can optionally +provide corresponding implementations to execute their own private processing. + +Uninstall UB Device Driver +-------------------------- +The UB device driver uses `ub_unregister_driver` to unregister the driver. This +interface call will perform a remove operation on all devices matched by the +driver, ultimately removing the UB device driver from the system. + +How to find UB devices manually +=============================== + +UBUS provides several interfaces to obtain ub_entities. You can search for them +using keywords such as GUID, EID, or entity number. Or you can find an entire +class of devices using vendor ID and device ID. + +How to access UB Configuration space +==================================== + +You can use `ub_config_(read|write)_(byte|word|dword)` to access the config +space of an entity represented by `struct ub_entity *`. All these functions return +0 when successful or an error code. Most drivers expect that accesses to valid UB +entities don't fail. + +The macros for configuration space registers are defined in the header file +include/uapi/ub/ubus/ubus_regs.h. + +Vendor and device identifications +================================= + +Do not add new device or vendor IDs to include/ub/ubus/ubus_ids.h unless they +are shared across multiple drivers. You can add private definitions in +your driver if they're helpful, or just use plain hex constants. + +The device IDs are arbitrary hex numbers (vendor controlled) and normally used +only in a single location, the ub_device_id table. + +Please DO submit new vendor/device IDs to . +There's a mirror of the ub.ids file at https://gitee.com/openeuler/ubutils/ub.ids. diff --git a/Documentation/ub/ummu/index.rst b/Documentation/ub/ummu/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..21360586e1f4ea520884cba0f7652471c6f2a158 --- /dev/null +++ b/Documentation/ub/ummu/index.rst @@ -0,0 +1,12 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +======================= +UB UMMU +======================= + +.. toctree:: + :maxdepth: 2 + :numbered: + + ummu + ummu-core diff --git a/Documentation/ub/ummu/ummu-core.rst b/Documentation/ub/ummu/ummu-core.rst new file mode 100644 index 0000000000000000000000000000000000000000..6a16bbaa641fc26724a09b77227805bf94363106 --- /dev/null +++ b/Documentation/ub/ummu/ummu-core.rst @@ -0,0 +1,128 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +====================================== +UMMU-CORE +====================================== + +:Authors: - Yanlong Zhu + +Introduction +============ +The Unified Bus Memory Management Unit (abbreviated as UMMU) is a component +that provides memory address mapping and access permission verification +during memory access processes. +It supports the sharing of memory resources between UBPU (UB Processing Units) +and ensures legitimate access to memory. + +The UMMU-Core is designed to work with the Linux IOMMU framework, as an +extension, providing the necessary interfaces to integrate with the system. +To maintain flexibility in deployment, the UMMU-Core can be compiled as a +loadable kernel module or built-in kernel image. + +EID Management +-------------- + +UMMU uses the following inputs — DstEID, TokenID and UBA (Unified Bus Address) — +to determine whether the entity is valid and which address domain it should access. + +Every UB entity must register its EID (Entity ID) with the UB domain to +communicate with other entities. UMMU-Core provides :c:func:`ummu_core_add_eid()` +and :c:func:`ummu_core_del_eid()` functions to manage EID. + +In some cases, UB devices may register before all UMMU devices. To handle +this, we designed an EID cached list to temporary save EIDs. Upon an UMMU +device register as global core device, the UMMU-Core will flushes the EID +cached list to it. Thread safety is guaranteed by the UMMU-Core. For +detailed information, refer to the `UB-Base-Specification-2.0`_. + +.. _UB-Base-Specification-2.0: https://www.unifiedbus.com/ + +TokenID Management +------------------ + +Each UB entity has multiple address spaces, such as DMA space, SVA space, +and others. The TokenID identifies the address space associated with each entity. + +The UMMU-Core introduces tdev (TID Device), a pseudo-device used to abstract +the concept of TID. It also supports UMMU driver functionality, enabling driver +management. The tdev can be used to allocate and grant memory address spaces. +When tdev is released, all associated resources will be freed. + +UMMU-Core acts as the TID manager in the UB system, offering TID allocation +strategies and TID allocation APIs to the UMMU driver. + +UMMU-Core supports multiple TID allocation strategies: + +- TRANSPARENT: + The TID is compatible with the global PASID (Process Address Space ID), + enabling seamless integration with system-wide address space management. +- ASSIGNED: + A pre-allocated TID, assigned from an external framework or management system. +- NORMAL: + The default TID allocation strategy, suitable for the majority of use cases. + +UMMU Device Registration +------------------------ + +The UMMU device registration is performed in two steps. An UMMU device +must implement the `ummu_core_device` interface and initialize it using +:c:func:`ummu_core_device_init()` function. This function initializes +the core device and allocates a dedicated TID manager to handle TID operations. + +Multiple UMMU devices can register to UMMU-Core by :c:func:`ummu_core_device_register()` +function. However, only global core device can take the charge of all UB device requests, +such as :c:func:`add_eid()` and :c:func:`del_eid()` functions. + +.. code-block:: none + + +-------------------+ + | IOMMU Framework | + +---------+---------+ + | + +----------+---------+ + | Global Core Device | + +----------+---------+ + | + +------------------------+-----------+-----------+------------------------+ + | | | | + +-------------------+ +-------------------+ +-------------------+ +-------------------+ + | Core Device 0 | | Core Device 1 | | Core Device 2 | ... | Core Device x | + +-------------------+ +-------------------+ +-------------------+ +-------------------+ + +Support KSVA mode +----------------- + +The KSVA (Kernel-space Shared Virtual Addressing) is not supported in the +current IOMMU framework, as it maps the entire kernel address space to +devices, which may cause critical errors. + +By leveraging isolated address space IDs and fine-grained permission controls, +we can restrict each device only access to the authorized address space +with KSVA mode. + +To manage the access permissions of each PASID, the IOMMU can implement a +permission checking mechanism. We abstract the permission management +operations into four fundamental types: + +- grant: + Grant access to a specified memory address range with defined + permissions (e.g., read, write, execute). +- ungrant: + Revoke previously granted access to a memory address range, invalidating + the device's permissions for that region. +- plb_sync_all: + Synchronize the PLB (Permission Lookaside Buffer) for all registered + PASIDs, ensuring global consistency of permission state across the IOMMU. +- plb_sync: + Synchronize the PLB for a specific PASID and memory range, minimizing + latency while maintaining access control integrity. + +These operations are integrated into the `iommu_domain` as part of the +`iommu_perm_ops` interface. + +UMMU SVA maintains a set of permission tables and page tables for each TID. +These resources can be allocated via the :c:func:`alloc_tid()` operation. +Once a TID is assigned, read and write permissions for the specific virtual +memory address ranges can be granted or ungranted. + +To access granted memory address ranges, permission verification is required. diff --git a/Documentation/ub/ummu/ummu.rst b/Documentation/ub/ummu/ummu.rst new file mode 100644 index 0000000000000000000000000000000000000000..b4f39749ff795d3ef1ecf8c0d8770a94c4550cb0 --- /dev/null +++ b/Documentation/ub/ummu/ummu.rst @@ -0,0 +1,134 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +=========== +UMMU Driver +=========== + +UMMU Functionality +================== + +The UMMU driver implements IOMMU functionality, enabling address +translation and access control for DMA transactions initiated by +peripheral devices. + +UMMU plays a critical role in system virtualization, device isolation, +and secure DMA address translation. + +In Shared Virtual Addressing (SVA) scenarios, UMMU enforces permission +checks to protect data within the shared address space, ensuring access +integrity and confidentiality. + +UMMU performs address translation and permission checking using input +parameters derived from the UB Memory Descriptor (EID + TokenID + UBA). + +For detailed information on the UB Memory Descriptor format and semantics, +refer to the `UB-Base-Specification-2.0`_. + +.. _UB-Base-Specification-2.0: https://www.unifiedbus.com/ + +The functionality of UMMU is primarily organized into the following three +core components: + +Configuration Table Lookup +-------------------------- + +The configuration data for address translation and permission checking in +UMMU is stored in memory and organized into two levels of configuration +tables: TECT (Target Entity Configuration Table) and TCT (Target Context +Table). + +- **TECT (Target Entity Configuration Table)**: + UMMU uses the DstEID to locate the corresponding TECT entry. This entry + primarily contains local entity information and serves as a storage + location for the entry points of the TCT and the Stage 2 address + translation tables. + +- **TCT (Target Context Table)**: + UMMU uses the TokenID to locate the corresponding TCT entry. This entry + describes the address space-level information, which may have a + granularity equal to or finer than that of the process level. The TCT + entry primarily stores the base addresses of the Stage 1 address + translation table and the MAPT (Memory Address Permission Table) used for + SVA mode permission checking. + +Address Translation +------------------- + +UMMU uses the EID and TokenID to locate the corresponding entries in the +TECT (Target Entity Configuration Table) and TCT (Target Context Table). +Based on the configuration table entries, it determines the base address +of the page table. It then uses the UBA and the page table base address to +perform the page table entry lookup and complete the address translation. + +In DMA scenarios, UMMU uses separate Stage 1 and Stage 2 translation +tables to support multiple-stage address translation. + +In user-space SVA scenarios, UMMU enables the device to directly access +the process's virtual address space. Similarly, kernel-space SVA allows +the device to access kernel-level virtual memory, enabling efficient data +sharing between the device and the kernel. + +Permission Checking +------------------- + +In SVA scenarios, UMMU performs permission checks to ensure the security +of the address space. + +UMMU performs permission checking in parallel with address translation. +After retrieving the TECT and TCT entries, if permission checking is +enabled for the currently accessed TECT entity, UMMU can obtain the MAPT +(Memory Address Permission Table) entry from the TCT entry. UMMU then +retrieves the permission information for the target memory from the MAPT, +compares it with the permissions specified in the memory access request, +and determines whether the access passes the permission check. + +The permission checking feature enables fine-grained control over memory +segment access, allowing the system to authorize or deauthorize specific +memory regions. It is recommended to enable the permission checking +feature to enforce security policies and protect the SVA address space +from unauthorized access. + +UMMU Driver Initialization +========================== + +When the UMMU driver detects an UMMU-capable platform device, it invokes +the probe function `ummu_device_probe()`. This function identifies the +device's hardware capabilities, allocates queues, configuration tables, +and interrupt handlers, and initializes the associated resources. + +UMMU Device Registration +======================== + +After the UMMU device completes its initialization, it is registered with +the UMMU framework. The UB system supports multiple UMMU devices within a +single chip. The UMMU framework abstracts a Logic UMMU device to uniformly +manage multiple physical UMMU devices. Once wrapped by the framework, the +Logic UMMU is ultimately registered with the IOMMU framework. + +In addition to calling the `struct iommu_ops` registered by individual UMMU +devices, the Logic UMMU leverages the extended operation set `struct +ummu_core_ops` provided by the UMMU framework to uniformly manage all +underlying UMMU device instances. This includes sharing configuration and +page table information across devices, and synchronizing invalidation +operations to ensure consistent table lookup results across the entire +device set. + +.. code-block:: none + + +-------------------+ + | IOMMU Framework | + +-------------------+ + ^ + | + Register + | + +--------------------+ + | UMMU-CORE Framework| + +--------------------+ + ^ + | + Register + | + +----------------+ +----------------+ +----------------+ + | ummu device 0 | | ummu device 1 | ... | ummu device x | + +----------------+ +----------------+ +----------------+ diff --git a/Documentation/ub/urma/udma/index.rst b/Documentation/ub/urma/udma/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..3a721ff1efccfaa662c9caf300056f8aa62fe786 --- /dev/null +++ b/Documentation/ub/urma/udma/index.rst @@ -0,0 +1,14 @@ +.. SPDX-License-Identifier: GPL-2.0+ +.. include:: + +:Copyright: |copy| 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + +============ +UDMA Driver +============ + +.. toctree:: + :maxdepth: 2 + :numbered: + + udma diff --git a/Documentation/ub/urma/udma/udma.rst b/Documentation/ub/urma/udma/udma.rst new file mode 100644 index 0000000000000000000000000000000000000000..8e17734fa580a52e0f4a901b87d3891c345f8300 --- /dev/null +++ b/Documentation/ub/urma/udma/udma.rst @@ -0,0 +1,296 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + +=============================================== +UnifiedBus DIRECT MEMORY ACCESS DRIVER (UDMA) +=============================================== + +Overview +========= + +This document describes the context and capabilities of the UDMA driver. + +**UnifiedBus** is an interconnect protocol for SuperPoD, It unifies IO, +memory access, and communication between various processing units under +a single interconnect technology framework. +The UnifiedBus specifications are open source and available on the official +website: `UB Specification Documents `_. + +**UDMA** (UnifiedBus Direct Memory Access), is a hardware I/O device that +provides direct memory access capabilities. + +**URMA(Unified Remote Memory Access)** is a component within the UnifiedBus +protocol stack, designed to abstract and facilitate communication between +different hardware and software entities. + +The UDMA driver integrates with the UnifiedBus protocol by implementing +the **URMA programming API**, through this API, the driver exposes the +UnifiedBus remote memory access programming model to application developers. + + +Device Driver Model +===================== + +The UDMA device is a UnifiedBus auxiliary device attached to the auxiliary bus. +The UDMA driver is developed based on the UBASE driver framework and uses +the auxiliary bus to perform device-driver binding. + +.. code-block:: none + + +---------------+ +-------+ + | UDMA Driver | | ... | + +-------+-------+ +-------+ + | + \|/ + +-------+-------------------+ + | auxiliary bus | + +-------+-------------------+ + /|\ + | + +-------+-------+ + | UDMA Device | + +----+----------+ + /|\ + | UBASE driver creates UDMA device + +----+------------------+ + | UBASE Driver | + +-----------+-----------+ + | + \|/ + +-----------+---------------+ + | ubus | + +---------------------------+ + +The figure above illustrates the hierarchy between the UDMA driver and the +UBASE driver. The UBASE driver is responsible for creating the UDMA auxiliary device +and registering it with the auxiliary bus. + + +Context & Submodules +======================= + +The UDMA driver depends on the ``Hardware programming interface``, +``UBASE driver``, and ``UMMU driver``. +It implements the URMA API and provides direct memory access capabilities. + +Below figure describe the UDMA driver's context and submodules. + +.. code-block:: none + + +-------------+ + | 5. URMA API | + +-----+-------+ + ^ + | + | + +-----------------+-----------------------+ + | UDMA Driver | + | | + | | + | +--------------------+ +-----------+ | + | | udma_main | | udma_comon| | + | +--------------------+ +-----------+ | + | +----------------++--------++--------+ | + | | udma_context ||udma_eid||udma_tid| | + | +----------------++--------++--------+ | + | +-----------------+ +----------------+ | +---------------+ + | | udma_jetty | | udma_segment | +----->| 4. UMMU Driver| + | +-----------------+ +----------------+ | +---------------+ + | +------------++---------++-----------+ | + | | udma_jfs ||udma_jfr || udma_jfc | | + | +------------++---------++-----------+ | + | +---------++---------------++--------+ | + | | udma_db || udma_ctrlq_tp ||udma_dfx| | + | +---------++---------------++--------+ | + | +-----------+ +---------+ +----------+ | + | | udma_cmd | |udma_ctl | | udma_eq | | + | +-----------+ +---------+ +----------+ | + +-----------------------------+-----------+ + | +---------------------+ + | | 3. Management Module| + \|/ +----------+----------+ + +--------+----------+ | + | 2. UBASE Driver +<---------------+ + +---------+---------+ + Software + -------------------------------+----------------------------------------- + \|/ Hardware + +-----------------------------+----------+ + | 1. Hardware programming interface | + +----------------------------------------+ + +Context +--------- + +1. Hardware programming interface: The UDMA driver encapsulates the + hardware programming interface, abstracting the hardware specifics. + +2. UBASE: UBASE driver responsible for managing UDMA auxiliary devices. + It also provides common management capabilities for auxiliary bus devices + and interacts with the Management module. + The UDMA device driver is built upon the UBASE driver and reuses its common utility functions. + +3. Management module: responsible for device management and configuration. + +4. UMMU: UnifiedBus Memory Management Unit, providing memory management + functionality(address translation, access permission, etc.) for UnifiedBus devices. + +5. URMA API: URMA programming interface, URMA API abstracts the memory operations, + and the UDMA driver implements it, so application developers do not need to be + aware of the details of the UDMA driver. + + +Submodules +------------ + +The UDMA driver submodules can be divided into 4 categories: +common utility and main functions, UDMA communication, device management and configuration, +UDMA device debugging. + +**Common Utility and Main Functions** + +* udma_main: Implements module_init/module_exit, and registers the UDMA driver + to the auxiliary bus. + +* udma_common: Provides common utility functions for UDMA driver. + +**UDMA Communication** + +Theses submodules handle UDMA communication setup and +processes(e.g read/write or send/recv operations). + +* udma_context: Manages UDMA communication context (allocates context, frees context, etc.). +* udma_eid: Manages UDMA Entity IDs (adds, removes, and queries UDMA entities). +* udma_tid: Manages TIDs (Token IDs) (allocates, frees token IDs). +* udma_segment: Manages memory segments, including local memory segment + registration and remote memory segment import. +* udma_jetty, udma_jfs, udma_jfr, udma_jfc: Manages UnifiedBus communication + jetty-related resources, including jetty, jfs, jfr, and jfc. + +**UDMA Device Management and Configuration** + +These submodules handle the UDMA device management and UDMA communication configuration. + +* udma_cmd: Encapsulates hardware configuration commands for UDMA communication, + e.g., create jfs, create jfc, etc. +* udma_db: Encapsulates UDMA hardware doorbell operations. +* udma_ctrlq_tp: Encapsulates control queue (CtrlQ) operations for UDMA hardware + configuration, e.g., get the transport channels. +* udma_ctl: Encapsulates UDMA hardware-specific configure operations, which are + not defined in the URMA API. Application developers should include the header file ``include/ub/urma/udma/udma_ctl.h`` separately. +* udma_eq: Encapsulates hardware event queue operations, e.g., register + CtrlQ event handle to receive CtrlQ events. + +**UDMA Device Debugging** + +* udma_dfx: Queries the UDMA hardware runtime configurations, e.g., + jetty state, transport mode, etc. + + +Supported Hardware +==================== + +UDMA driver supported hardware: + +=========== ============= +Vendor ID Device ID +=========== ============= +0xCC08 0xA001 +0xCC08 0xA002 +0xCC08 0xD802 +0xCC08 0xD803 +0xCC08 0xD80B +0xCC08 0xD80C +=========== ============= + +You can use the ``lsub`` command on your host OS to query UB devices. Below is an example output: + +.. code-block:: shell + + UB network controller <0002>: Huawei Technologies Co., Ltd. URMA management ub entity : + UB network controller <0082>: Huawei Technologies Co., Ltd. URMA management ub entity : + UB network controller <0002>: Huawei Technologies Co., Ltd. URMA management ub entity : + +The ``Vendor ID`` and ``Device ID`` are located at the end of each output line +with the format ``:``, e.g., ``:``. + +Note the ``lsub`` command is from ubutils; make sure it is installed on your host. + + +Module Parameters +=================== + +UDMA driver supports 4 parameters: **cqe_mode**, **jfc_arm_mode**, +**jfr_sleep_time**, **dump_aux_info**. +The default value represents the best practices; however, you may need to change +the default value in certain cases. + +cqe_mode +----------- + +``cqe_mode`` controls the method of **Completion Queue Entry (CQE)** event generation. + +In interrupt mode, UDMA provides two mechanisms for generating CQE events: +**Producer Index (PI)/Consumer Index (CI) difference comparison** +and **counter-based threshold**. + +* PI/CI difference comparison: PI (Producer Index) and CI (Consumer Index) + respectively point to the next CQE to be written and read in the Completion Queue. + The device generates an interrupt to notify the upper layer when the + difference (the number of pending CQEs) exceeds a certain threshold. +* Counter-based threshold: An interrupt is generated when the total number of + CQEs written to the Completion Queue reaches a programmed threshold. + +**Parameter values:** + +* 0: Counter-based threshold +* 1: PI/CI difference comparison + +**Default value**: 1 + + +jfc_arm_mode +-------------- + +`jfc_arm_mode` controls the completion event interrupt mode. + +**Parameter Values:** + +* 0: Always ARM, interrupt always enabled +* 1: No ARM, interrupt is disabled and cannot be modified +* Other value (e.g., 2): Interrupt is disabled, but can be modified + +**Default value:** 0 + + +jfr_sleep_time +---------------- + +``jfr_sleep_time`` configures the maximum blocking wait time (in microseconds) +when deregistering a JFR (Jetty-related resource). The default value is 1000 us. +You can adjust this parameter value as needed. + +The allowed range is: ``[0,UINT32_MAX]`` + +dump_aux_info +--------------- + +``dump_aux_info`` controls whether to dump auxiliary information +(the hardware register values) into the event body when reporting asynchronous +or completion events. + + +**Parameter Values:** + +* false: Disables the dumping of auxiliary information. +* true: Enables the dumping of auxiliary information. + +**Default value**: false + + +Support +======= + +If there is any issue or question, please email the specific information related +to the issue or question to or vendor's support channel. diff --git a/Documentation/userspace-api/fwctl/fwctl.rst b/Documentation/userspace-api/fwctl/fwctl.rst new file mode 100644 index 0000000000000000000000000000000000000000..8c4472f980658a01e7ddefd2c8a707bfb5ce1c6c --- /dev/null +++ b/Documentation/userspace-api/fwctl/fwctl.rst @@ -0,0 +1,285 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=============== +fwctl subsystem +=============== + +:Author: Jason Gunthorpe + +Overview +======== + +Modern devices contain extensive amounts of FW, and in many cases, are largely +software-defined pieces of hardware. The evolution of this approach is largely a +reaction to Moore's Law where a chip tape out is now highly expensive, and the +chip design is extremely large. Replacing fixed HW logic with a flexible and +tightly coupled FW/HW combination is an effective risk mitigation against chip +respin. Problems in the HW design can be counteracted in device FW. This is +especially true for devices which present a stable and backwards compatible +interface to the operating system driver (such as NVMe). + +The FW layer in devices has grown to incredible size and devices frequently +integrate clusters of fast processors to run it. For example, mlx5 devices have +over 30MB of FW code, and big configurations operate with over 1GB of FW managed +runtime state. + +The availability of such a flexible layer has created quite a variety in the +industry where single pieces of silicon are now configurable software-defined +devices and can operate in substantially different ways depending on the need. +Further, we often see cases where specific sites wish to operate devices in ways +that are highly specialized and require applications that have been tailored to +their unique configuration. + +Further, devices have become multi-functional and integrated to the point they +no longer fit neatly into the kernel's division of subsystems. Modern +multi-functional devices have drivers, such as bnxt/ice/mlx5/pds, that span many +subsystems while sharing the underlying hardware using the auxiliary device +system. + +All together this creates a challenge for the operating system, where devices +have an expansive FW environment that needs robust device-specific debugging +support, and FW-driven functionality that is not well suited to “generic” +interfaces. fwctl seeks to allow access to the full device functionality from +user space in the areas of debuggability, management, and first-boot/nth-boot +provisioning. + +fwctl is aimed at the common device design pattern where the OS and FW +communicate via an RPC message layer constructed with a queue or mailbox scheme. +In this case the driver will typically have some layer to deliver RPC messages +and collect RPC responses from device FW. The in-kernel subsystem drivers that +operate the device for its primary purposes will use these RPCs to build their +drivers, but devices also usually have a set of ancillary RPCs that don't really +fit into any specific subsystem. For example, a HW RAID controller is primarily +operated by the block layer but also comes with a set of RPCs to administer the +construction of drives within the HW RAID. + +In the past when devices were more single function, individual subsystems would +grow different approaches to solving some of these common problems. For instance +monitoring device health, manipulating its FLASH, debugging the FW, +provisioning, all have various unique interfaces across the kernel. + +fwctl's purpose is to define a common set of limited rules, described below, +that allow user space to securely construct and execute RPCs inside device FW. +The rules serve as an agreement between the operating system and FW on how to +correctly design the RPC interface. As a uAPI the subsystem provides a thin +layer of discovery and a generic uAPI to deliver the RPCs and collect the +response. It supports a system of user space libraries and tools which will +use this interface to control the device using the device native protocols. + +Scope of Action +--------------- + +fwctl drivers are strictly restricted to being a way to operate the device FW. +It is not an avenue to access random kernel internals, or other operating system +SW states. + +fwctl instances must operate on a well-defined device function, and the device +should have a well-defined security model for what scope within the physical +device the function is permitted to access. For instance, the most complex PCIe +device today may broadly have several function-level scopes: + + 1. A privileged function with full access to the on-device global state and + configuration + + 2. Multiple hypervisor functions with control over itself and child functions + used with VMs + + 3. Multiple VM functions tightly scoped within the VM + +The device may create a logical parent/child relationship between these scopes. +For instance a child VM's FW may be within the scope of the hypervisor FW. It is +quite common in the VFIO world that the hypervisor environment has a complex +provisioning/profiling/configuration responsibility for the function VFIO +assigns to the VM. + +Further, within the function, devices often have RPC commands that fall within +some general scopes of action (see enum fwctl_rpc_scope): + + 1. Access to function & child configuration, FLASH, etc. that becomes live at a + function reset. Access to function & child runtime configuration that is + transparent or non-disruptive to any driver or VM. + + 2. Read-only access to function debug information that may report on FW objects + in the function & child, including FW objects owned by other kernel + subsystems. + + 3. Write access to function & child debug information strictly compatible with + the principles of kernel lockdown and kernel integrity protection. Triggers + a kernel Taint. + + 4. Full debug device access. Triggers a kernel Taint, requires CAP_SYS_RAWIO. + +User space will provide a scope label on each RPC and the kernel must enforce the +above CAPs and taints based on that scope. A combination of kernel and FW can +enforce that RPCs are placed in the correct scope by user space. + +Denied behavior +--------------- + +There are many things this interface must not allow user space to do (without a +Taint or CAP), broadly derived from the principles of kernel lockdown. Some +examples: + + 1. DMA to/from arbitrary memory, hang the system, compromise FW integrity with + untrusted code, or otherwise compromise device or system security and + integrity. + + 2. Provide an abnormal “back door” to kernel drivers. No manipulation of kernel + objects owned by kernel drivers. + + 3. Directly configure or otherwise control kernel drivers. A subsystem kernel + driver can react to the device configuration at function reset/driver load + time, but otherwise must not be coupled to fwctl. + + 4. Operate the HW in a way that overlaps with the core purpose of another + primary kernel subsystem, such as read/write to LBAs, send/receive of + network packets, or operate an accelerator's data plane. + +fwctl is not a replacement for device direct access subsystems like uacce or +VFIO. + +Operations exposed through fwctl's non-taining interfaces should be fully +sharable with other users of the device. For instance exposing a RPC through +fwctl should never prevent a kernel subsystem from also concurrently using that +same RPC or hardware unit down the road. In such cases fwctl will be less +important than proper kernel subsystems that eventually emerge. Mistakes in this +area resulting in clashes will be resolved in favour of a kernel implementation. + +fwctl User API +============== + +.. kernel-doc:: include/uapi/fwctl/fwctl.h +.. kernel-doc:: include/uapi/fwctl/ub_fwctl.h + +sysfs Class +----------- + +fwctl has a sysfs class (/sys/class/fwctl/fwctlNN/) and character devices +(/dev/fwctl/fwctlNN) with a simple numbered scheme. The character device +operates the iotcl uAPI described above. + +fwctl devices can be related to driver components in other subsystems through +sysfs:: + + $ ls /sys/class/fwctl/fwctl0/device/infiniband/ + ibp0s10f0 + + $ ls /sys/class/infiniband/ibp0s10f0/device/fwctl/ + fwctl0/ + + $ ls /sys/devices/pci0000:00/0000:00:0a.0/fwctl/fwctl0 + dev device power subsystem uevent + +User space Community +-------------------- + +Drawing inspiration from nvme-cli, participating in the kernel side must come +with a user space in a common TBD git tree, at a minimum to usefully operate the +kernel driver. Providing such an implementation is a pre-condition to merging a +kernel driver. + +The goal is to build user space community around some of the shared problems +we all have, and ideally develop some common user space programs with some +starting themes of: + + - Device in-field debugging + + - HW provisioning + + - VFIO child device profiling before VM boot + + - Confidential Compute topics (attestation, secure provisioning) + +that stretch across all subsystems in the kernel. fwupd is a great example of +how an excellent user space experience can emerge out of kernel-side diversity. + +fwctl Kernel API +================ + +.. kernel-doc:: drivers/fwctl/main.c + :export: +.. kernel-doc:: include/linux/fwctl.h + +fwctl Driver design +------------------- + +In many cases a fwctl driver is going to be part of a larger cross-subsystem +device possibly using the auxiliary_device mechanism. In that case several +subsystems are going to be sharing the same device and FW interface layer so the +device design must already provide for isolation and cooperation between kernel +subsystems. fwctl should fit into that same model. + +Part of the driver should include a description of how its scope restrictions +and security model work. The driver and FW together must ensure that RPCs +provided by user space are mapped to the appropriate scope. If the validation is +done in the driver then the validation can read a 'command effects' report from +the device, or hardwire the enforcement. If the validation is done in the FW, +then the driver should pass the fwctl_rpc_scope to the FW along with the command. + +The driver and FW must cooperate to ensure that either fwctl cannot allocate +any FW resources, or any resources it does allocate are freed on FD closure. A +driver primarily constructed around FW RPCs may find that its core PCI function +and RPC layer belongs under fwctl with auxiliary devices connecting to other +subsystems. + +Each device type must be mindful of Linux's philosophy for stable ABI. The FW +RPC interface does not have to meet a strictly stable ABI, but it does need to +meet an expectation that userspace tools that are deployed and in significant +use don't needlessly break. FW upgrade and kernel upgrade should keep widely +deployed tooling working. + +Development and debugging focused RPCs under more permissive scopes can have +less stabilitiy if the tools using them are only run under exceptional +circumstances and not for every day use of the device. Debugging tools may even +require exact version matching as they may require something similar to DWARF +debug information from the FW binary. + +Security Response +================= + +The kernel remains the gatekeeper for this interface. If violations of the +scopes, security or isolation principles are found, we have options to let +devices fix them with a FW update, push a kernel patch to parse and block RPC +commands or push a kernel patch to block entire firmware versions/devices. + +While the kernel can always directly parse and restrict RPCs, it is expected +that the existing kernel pattern of allowing drivers to delegate validation to +FW to be a useful design. + +Existing Similar Examples +========================= + +The approach described in this document is not a new idea. Direct, or near +direct device access has been offered by the kernel in different areas for +decades. With more devices wanting to follow this design pattern it is becoming +clear that it is not entirely well understood and, more importantly, the +security considerations are not well defined or agreed upon. + +Some examples: + + - HW RAID controllers. This includes RPCs to do things like compose drives into + a RAID volume, configure RAID parameters, monitor the HW and more. + + - Baseboard managers. RPCs for configuring settings in the device and more + + - NVMe vendor command capsules. nvme-cli provides access to some monitoring + functions that different products have defined, but more exist. + + - CXL also has a NVMe-like vendor command system. + + - DRM allows user space drivers to send commands to the device via kernel + mediation + + - RDMA allows user space drivers to directly push commands to the device + without kernel involvement + + - Various “raw” APIs, raw HID (SDL2), raw USB, NVMe Generic Interface, etc. + +The first 4 are examples of areas that fwctl intends to cover. The latter three +are examples of denied behavior as they fully overlap with the primary purpose +of a kernel subsystem. + +Some key lessons learned from these past efforts are the importance of having a +common user space project to use as a pre-condition for obtaining a kernel +driver. Developing good community around useful software in user space is key to +getting companies to fund participation to enable their products. diff --git a/Documentation/userspace-api/fwctl/index.rst b/Documentation/userspace-api/fwctl/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..be74da876caea865bdbe50dd27b18e6a8a94f1b6 --- /dev/null +++ b/Documentation/userspace-api/fwctl/index.rst @@ -0,0 +1,13 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Firmware Control (FWCTL) Userspace API +====================================== + +A framework that define a common set of limited rules that allows user space +to securely construct and execute RPCs inside device firmware. + +.. toctree:: + :maxdepth: 1 + + fwctl + ub_fwctl diff --git a/Documentation/userspace-api/fwctl/ub_fwctl.rst b/Documentation/userspace-api/fwctl/ub_fwctl.rst new file mode 100644 index 0000000000000000000000000000000000000000..bdc0d3a5a7b68a87269345dfa289df19c3faf320 --- /dev/null +++ b/Documentation/userspace-api/fwctl/ub_fwctl.rst @@ -0,0 +1,51 @@ +.. SPDX-License-Identifier: GPL-2.0 + +================ +fwctl ub driver +================ + +Overview +======== + +The ub_fwctl tool is primarily designed to provide functions including querying +the configuration of UB common functions, the status and statistics of common modules, +and information at the Die level. Ub_fwctl is integrated with the open-source fwtl framework, +providing a custom user-mode command format for UB and supporting the common functionality of UB systems. + +The implemented driver is ub_fwctl, which includes the user-mode command line +tool ubctl and kernel-mode driver ub_fwctl. After the ub_fwctl driver is loaded, +a file such as ub_ctl is displayed in the /dev directory of the OS through the +sysfs system. The user-mode program ubtl obtains file descriptors by calling +open (/dev/fwctl/fwctlNN), and then communicates with the driver by calling ioctl. + +Function implementation scheme:: + + 1. Ub_fwctl registers itself with the fwctl framework. + + 2. As an auxiliary device, ub_fwctl is connected to ubase through an + auxiliary bus and uses pmu idev's CMDQ to call the software programming + interface to read and write registers. + + 3. Ubctl provides command-line commands for users to call. At startup, + ubctl opens the ubctl device file, assembles the corresponding data + structure based on input, and calls ioctl to enter kernel state. After + receiving the ubctl command, ub_fwctl first checks the legality of the + command, and then communicates with the software by calling the interface + provided by ubase to access CMDQ. The software returns the result of accessing + the register to ub_fwctl through CMDQ, and ub_fwctl then returns the + data to user state. Finally, close the opened ubctl file. + +ub_fwctl User API +================== + +First step for the app is to issue the ioctl(UBCTL_IOCTL_CMDRPC). Each RPC +request includes the operation id, and in and out buffer lengths and pointers. +The driver verifies the operations, then checks the request scope against the +required scope of the operation. The request is then put together with the +request data and sent through the software's message queue to the firmware, and the +results are returned to the caller. + +The RPC endpoints, operations, and buffer contents are defined by the +particular firmware package in the device, which varies across the +available product configurations. The details are available in the +specific product SDK documentation. diff --git a/Documentation/userspace-api/index.rst b/Documentation/userspace-api/index.rst index 72a65db0c498898749d6e182e2e277ebd7647291..c02b2bc235dbcdfe7d87a358cd16e89d7a3b30de 100644 --- a/Documentation/userspace-api/index.rst +++ b/Documentation/userspace-api/index.rst @@ -26,12 +26,14 @@ place where this information is gathered. ELF ioctl/index iommu + fwctl/index iommufd media/index netlink/index sysfs-platform_profile vduse futex2 + ub/index .. only:: subproject and html diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index 4ea5b837399ad1cb93f837a0f090bc87ed1f3089..66ad52639a2887087b9ad13ea200b2586736ce08 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -321,6 +321,7 @@ Code Seq# Include File Comments 0x97 00-7F fs/ceph/ioctl.h Ceph file system 0x99 00-0F 537-Addinboard driver +0x9A 00-0F include/uapi/fwctl/fwctl.h 0xA0 all linux/sdp/sdp.h Industrial Device Project 0xA1 0 linux/vtpm_proxy.h TPM Emulator Proxy Driver diff --git a/Documentation/userspace-api/ub/cdma.rst b/Documentation/userspace-api/ub/cdma.rst new file mode 100644 index 0000000000000000000000000000000000000000..e5b9a1e9de76409293de19eafeade1ff40eb92c2 --- /dev/null +++ b/Documentation/userspace-api/ub/cdma.rst @@ -0,0 +1,51 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + +=============================== +CDMA Userspace Support Library +=============================== + +Overview +========= +CDMA (Crystal Direct Memory Access) is used to provide asynchronous memory read +and write operations between hosts or between host and devices. + +The key features are described as follows: + ++ 1. Peer-to-peer communication between hosts, enabling bidirectional asynchronous memory read or write. ++ 2. Asynchronous memory read and write between host and devices via DMA. ++ 3. Asynchronous memory read and write between devices and host via DMA. + +Char Device +============= +The driver creates one char device per CDMA found on the physical device. +Char devices can be found in /dev/cdma/ and are named as: +/dev/cdma/dev. + +User API +========= + +ioctl +------ +========================= ==================================================== +CDMA_CMD_QUERY_DEV_INFO Query CDMA device information. +CDMA_CMD_CREATE_CTX Create user context resource. +CDMA_CMD_DELETE_CTX Delete user context resource. +CDMA_CMD_CREATE_CTP Create CTP(Compact Transport) channel resource. +CDMA_CMD_DELETE_CTP Delete CTP channel resource. +CDMA_CMD_CREATE_JFS Create JFS(Jetty For Send) resource. +CDMA_CMD_DELETE_JFS Delete JFS resource. +CDMA_CMD_REGISTER_SEG Register local segment resource. +CDMA_CMD_UNREGISTER_SEG Unregister local segment resource. +CDMA_CMD_CREATE_QUEUE Create queue resource. +CDMA_CMD_DELETE_QUEUE Delete queue resource. +CDMA_CMD_CREATE_JFC Create JFC(Jetty For Completion) resource. +CDMA_CMD_DELETE_JFC Delete JFC resource. +CDMA_CMD_CREATE_JFCE Create JFCE(Jetty For Completion Event) resource. +========================= ==================================================== + +Support +======== +If there is any issue or question, please email the specific information related +to the issue or question to or vendor's support channel. \ No newline at end of file diff --git a/Documentation/userspace-api/ub/index.rst b/Documentation/userspace-api/ub/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..3206a2cf64c71ef2367e1638a57e2a485882c2a2 --- /dev/null +++ b/Documentation/userspace-api/ub/index.rst @@ -0,0 +1,13 @@ +.. SPDX-License-Identifier: GPL-2.0+ +.. include:: + +:Copyright: |copy| 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + +===================== +UnifiedBus Subsystem +===================== + +.. toctree:: + :maxdepth: 1 + + cdma \ No newline at end of file diff --git a/Documentation/userspace-api/ummu_core.rst b/Documentation/userspace-api/ummu_core.rst new file mode 100644 index 0000000000000000000000000000000000000000..79d9dd7a574018127c6f974dfe276313f6a05566 --- /dev/null +++ b/Documentation/userspace-api/ummu_core.rst @@ -0,0 +1,103 @@ +.. SPDX-License-Identifier: GPL-2.0+ +.. ummu_core: + +======================= +UMMU_CORE Userspace API +======================= + +The UMMU UAPI provides APIs that enable communication between user-space components +and kernel-space components.The primary use case is Shared Virtual Address (SVA). + +.. contents:: :local: + +Functionalities +=============== +Only kernel-mode process expose the APIs. The supported user-kernel APIs +are as follows: + +1. Allocate/Free a TID +2. Send one or more PLBI commands +3. Map or unmap resources, including MAPT block and command queues + +Interfaces +========== +Although the data structures defined in UMMU_CORE UAPI are self-contained, +no user-facing API functions are provided. Instead, UMMU_CORE UAPI is +designed to work with UMMU_CORE driver. + +Upon loading, the UMMU_CORE driver registers a TID device, and sets up its operation function table. +The supported operations include open, release, map, and ioctl. + +Datastructures and Definitions +------------------------------ +1. struct ummu_token_info: stores token information for a shared-memory segment. + + - input: specifies the token generation mode.If input is 0, the tokenVal field is used as the token value. + If input is 1, the UMMU library generates a random token value, and tokenVal is ignored. + - tokenVal: the token value to use when input is 0. + +2. enum ummu_mapt_perm: access permissions for a shared-memory segment + + - MAPT_PERM_W: write only + - MAPT_PERM_R: read only + - MAPT_PERM_RW: read and write + - MAPT_PERM_ATOMIC_W: atomic write only + - MAPT_PERM_ATOMIC_R: atomic read only + - MAPT_PERM_ATOMIC_RW: atomic read and write + +3. enum ummu_mapt_mode: Memory Address Permission Table mode + + - MAPT_MODE_ENTRY: only one memory address segment can be managed per TID. + - MAPT_MODE_TABLE: multiple memory address segments can be managed per TID. + +4. enum ummu_ebit_state: + + - UMMU_EBIT_OFF: disable ebit check + - UMMU_EBIT_ON: enable ebit check + +5. definitions: + + - TID_DEVICE_NAME: a character device that enables user-mode processes to interact + with hardware or software through system calls. + - UMMU_IOCALLOC_TID: operation code for allocating a TID. + - UMMU_IOCFREE_TID: operation code for freeing a TID. + - UMMU_IOCPLBI_VA: operation code to flush the PLB cache for a specific virtual address. + - UMMU_IOCPLBI_ALL: operation code to flush the PLB cache for all virtual addresses. + +Descriptions and Examples +------------------------- +1. allocate/free tid + +The input parameters is *struct ummu_tid_info*.Below is an example: +:: + struct ummu_tid_info info = {}; + + int fd = open("/dev/ummu/tid", O_RDWR | O_CLOEXEC); + ioctl(fd, UMMU_IOCALLOC_TID, &info); + ioctl(fd, UMMU_IOCFREE_TID, &info); + +The PLBI command operation is performed via the ioctl interface, +using the operation codes UMMU_IOCPLBI_VA or UMMU_IOCPLBI_ALL. + +2. map resources + +This interface is used in two scenarios: +(1) Creating a new MAPT block +(2) Initializing user-mode queues + +For example: +:: + mmap(NULL, size, prot, flags, fd, PA); + +On success, this returns a virtual address. + +3. unmap resources + +This interface is used in two scenarios: +(1) Clearing MAPT blocks +(2) When user-mode process exits, all associated MAPT blocks and use-mode queue resources +are cleared. + +For example: +:: + munmap(buf, BLOCK_SIZE_4K); diff --git a/MAINTAINERS b/MAINTAINERS index 29ee45a167d148cfc94afba6f11e15adad30c350..cac72e421f1ad6a206813ce5e365bb687599ac85 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8640,6 +8640,17 @@ F: kernel/futex/* F: tools/perf/bench/futex* F: tools/testing/selftests/futex/ +FWCTL SUBSYSTEM +M: Dave Jiang +M: Jason Gunthorpe +M: Saeed Mahameed +R: Jonathan Cameron +S: Maintained +F: Documentation/userspace-api/fwctl/ +F: drivers/fwctl/ +F: include/linux/fwctl.h +F: include/uapi/fwctl/ + GATEWORKS SYSTEM CONTROLLER (GSC) DRIVER M: Tim Harvey S: Maintained diff --git a/arch/arm64/configs/tencent.config b/arch/arm64/configs/tencent.config index a8f6f34cdee62c7f2296617c1d9443c761aeebf2..7ca1fa88f87735d4de4f009259e872f167fbe717 100644 --- a/arch/arm64/configs/tencent.config +++ b/arch/arm64/configs/tencent.config @@ -1834,6 +1834,12 @@ CONFIG_UB_UBL=m CONFIG_UB_UNIC=m CONFIG_UB_UNIC_UBL=y CONFIG_UB_UNIC_DCB=y + +# UB CDMA driver +CONFIG_UB_CDMA=m + +# UB UDMA driver +CONFIG_UB_UDMA=m # end of unified bus # UMMU @@ -1860,3 +1866,10 @@ CONFIG_UB_URMA=m CONFIG_IOMMUFD=m CONFIG_VFIO_DEVICE_CDEV=y # end of IOMMUFD + +# fwctl +CONFIG_FWCTL=m + +# UB_FWCTL +CONFIG_FWCTL_UB=m +# end of UB_FWCTL diff --git a/arch/x86/configs/tencent.config b/arch/x86/configs/tencent.config index 8958dc6d45277817e7948759cd32c8020e77a879..2444a388a8c19bae3ef4b3da281d84b0048f4938 100644 --- a/arch/x86/configs/tencent.config +++ b/arch/x86/configs/tencent.config @@ -2026,3 +2026,6 @@ CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_KSTRTOX=y CONFIG_TEST_BPF=m CONFIG_EXT4_FS=y + +# fwctl +CONFIG_FWCTL=m diff --git a/drivers/Kconfig b/drivers/Kconfig index bfb2bdb004774af576d4f32bdcc5ac712e1f74de..1a9785701376bc6066d893f1bdd2998b81509bc3 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -22,6 +22,8 @@ source "drivers/connector/Kconfig" source "drivers/firmware/Kconfig" +source "drivers/fwctl/Kconfig" + source "drivers/gnss/Kconfig" source "drivers/mtd/Kconfig" diff --git a/drivers/Makefile b/drivers/Makefile index 800793aafbbb20ee149daa4ff6479eaf9af0982f..269267ac3b4fb44012b9e74a4060738b908d04e8 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -141,6 +141,7 @@ obj-$(CONFIG_MEMSTICK) += memstick/ obj-$(CONFIG_NEW_LEDS) += leds/ obj-$(CONFIG_INFINIBAND) += infiniband/ obj-y += firmware/ +obj-$(CONFIG_FWCTL) += fwctl/ obj-$(CONFIG_CRYPTO) += crypto/ obj-$(CONFIG_SUPERH) += sh/ obj-y += clocksource/ diff --git a/drivers/fwctl/Kconfig b/drivers/fwctl/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..88b346631ebead583e95348f27bdb066718ceba2 --- /dev/null +++ b/drivers/fwctl/Kconfig @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0-only +menuconfig FWCTL + tristate "fwctl device firmware access framework" + help + fwctl provides a userspace API for restricted access to communicate + with on-device firmware. The communication channel is intended to + support a wide range of lockdown compatible device behaviors including + manipulating device FLASH, debugging, and other activities that don't + fit neatly into an existing subsystem. + +if FWCTL +config FWCTL_UB + tristate "ub_fwctl depend on fwctl driver" + depends on UB_UBASE + default n + help + ub_fwctl provides users with various information related to + querying UB (UnifiedBus) registers or devices. + + If you don't know what to do here, say N. +endif diff --git a/drivers/fwctl/Makefile b/drivers/fwctl/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..9005cdf31dfdb585ccd80b49f2b60294c6532e7f --- /dev/null +++ b/drivers/fwctl/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_FWCTL) += fwctl.o +obj-$(CONFIG_FWCTL_UB) += ub/ + +fwctl-y += main.o diff --git a/drivers/fwctl/main.c b/drivers/fwctl/main.c new file mode 100644 index 0000000000000000000000000000000000000000..7854ab0369f2d3f5e4c094a3b96d09eb4e7d46f8 --- /dev/null +++ b/drivers/fwctl/main.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES + */ +#define pr_fmt(fmt) "fwctl: " fmt +#include + +#include +#include +#include +#include +#include + +#include + +enum { + FWCTL_MAX_DEVICES = 4096, + MAX_RPC_LEN = SZ_2M, +}; +static_assert(FWCTL_MAX_DEVICES < (1U << MINORBITS)); + +static dev_t fwctl_dev; +static DEFINE_IDA(fwctl_ida); +static unsigned long fwctl_tainted; + +struct fwctl_ucmd { + struct fwctl_uctx *uctx; + void __user *ubuffer; + void *cmd; + u32 user_size; +}; + +static int ucmd_respond(struct fwctl_ucmd *ucmd, size_t cmd_len) +{ + if (copy_to_user(ucmd->ubuffer, ucmd->cmd, + min_t(size_t, ucmd->user_size, cmd_len))) + return -EFAULT; + return 0; +} + +static int copy_to_user_zero_pad(void __user *to, const void *from, + size_t from_len, size_t user_len) +{ + size_t copy_len; + + copy_len = min(from_len, user_len); + if (copy_to_user(to, from, copy_len)) + return -EFAULT; + if (copy_len < user_len) { + if (clear_user(to + copy_len, user_len - copy_len)) + return -EFAULT; + } + return 0; +} + +static int fwctl_cmd_info(struct fwctl_ucmd *ucmd) +{ + struct fwctl_device *fwctl = ucmd->uctx->fwctl; + struct fwctl_info *cmd = ucmd->cmd; + size_t driver_info_len = 0; + + if (cmd->flags) + return -EOPNOTSUPP; + + if (!fwctl->ops->info && cmd->device_data_len) { + if (clear_user(u64_to_user_ptr(cmd->out_device_data), + cmd->device_data_len)) + return -EFAULT; + } else if (cmd->device_data_len) { + void *driver_info __free(kfree) = + fwctl->ops->info(ucmd->uctx, &driver_info_len); + if (IS_ERR(driver_info)) + return PTR_ERR(driver_info); + + if (copy_to_user_zero_pad(u64_to_user_ptr(cmd->out_device_data), + driver_info, driver_info_len, + cmd->device_data_len)) + return -EFAULT; + } + + cmd->out_device_type = fwctl->ops->device_type; + cmd->device_data_len = driver_info_len; + return ucmd_respond(ucmd, sizeof(*cmd)); +} + +static int fwctl_cmd_rpc(struct fwctl_ucmd *ucmd) +{ + struct fwctl_device *fwctl = ucmd->uctx->fwctl; + struct fwctl_rpc *cmd = ucmd->cmd; + size_t out_len; + + if (cmd->in_len > MAX_RPC_LEN || cmd->out_len > MAX_RPC_LEN) + return -EMSGSIZE; + + switch (cmd->scope) { + case FWCTL_RPC_CONFIGURATION: + case FWCTL_RPC_DEBUG_READ_ONLY: + break; + + case FWCTL_RPC_DEBUG_WRITE_FULL: + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; + fallthrough; + case FWCTL_RPC_DEBUG_WRITE: + if (!test_and_set_bit(0, &fwctl_tainted)) { + dev_warn( + &fwctl->dev, + "%s(%d): has requested full access to the physical device", + current->comm, task_pid_nr(current)); + add_taint(TAINT_FWCTL, LOCKDEP_STILL_OK); + } + break; + default: + return -EOPNOTSUPP; + } + + void *inbuf __free(kvfree) = kvzalloc(cmd->in_len, GFP_KERNEL_ACCOUNT); + if (!inbuf) + return -ENOMEM; + if (copy_from_user(inbuf, u64_to_user_ptr(cmd->in), cmd->in_len)) + return -EFAULT; + + out_len = cmd->out_len; + void *outbuf __free(kvfree) = fwctl->ops->fw_rpc( + ucmd->uctx, cmd->scope, inbuf, cmd->in_len, &out_len); + if (IS_ERR(outbuf)) + return PTR_ERR(outbuf); + if (outbuf == inbuf) { + /* The driver can re-use inbuf as outbuf */ + inbuf = NULL; + } + + if (copy_to_user(u64_to_user_ptr(cmd->out), outbuf, + min(cmd->out_len, out_len))) + return -EFAULT; + + cmd->out_len = out_len; + return ucmd_respond(ucmd, sizeof(*cmd)); +} + +/* On stack memory for the ioctl structs */ +union fwctl_ucmd_buffer { + struct fwctl_info info; + struct fwctl_rpc rpc; +}; + +struct fwctl_ioctl_op { + unsigned int size; + unsigned int min_size; + unsigned int ioctl_num; + int (*execute)(struct fwctl_ucmd *ucmd); +}; + +#define IOCTL_OP(_ioctl, _fn, _struct, _last) \ + [_IOC_NR(_ioctl) - FWCTL_CMD_BASE] = { \ + .size = sizeof(_struct) + \ + BUILD_BUG_ON_ZERO(sizeof(union fwctl_ucmd_buffer) < \ + sizeof(_struct)), \ + .min_size = offsetofend(_struct, _last), \ + .ioctl_num = _ioctl, \ + .execute = _fn, \ + } +static const struct fwctl_ioctl_op fwctl_ioctl_ops[] = { + IOCTL_OP(FWCTL_INFO, fwctl_cmd_info, struct fwctl_info, out_device_data), + IOCTL_OP(FWCTL_RPC, fwctl_cmd_rpc, struct fwctl_rpc, out), +}; + +static long fwctl_fops_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) +{ + struct fwctl_uctx *uctx = filp->private_data; + const struct fwctl_ioctl_op *op; + struct fwctl_ucmd ucmd = {}; + union fwctl_ucmd_buffer buf; + unsigned int nr; + int ret; + + nr = _IOC_NR(cmd); + if ((nr - FWCTL_CMD_BASE) >= ARRAY_SIZE(fwctl_ioctl_ops)) + return -ENOIOCTLCMD; + + op = &fwctl_ioctl_ops[nr - FWCTL_CMD_BASE]; + if (op->ioctl_num != cmd) + return -ENOIOCTLCMD; + + ucmd.uctx = uctx; + ucmd.cmd = &buf; + ucmd.ubuffer = (void __user *)arg; + ret = get_user(ucmd.user_size, (u32 __user *)ucmd.ubuffer); + if (ret) + return ret; + + if (ucmd.user_size < op->min_size) + return -EINVAL; + + ret = copy_struct_from_user(ucmd.cmd, op->size, ucmd.ubuffer, + ucmd.user_size); + if (ret) + return ret; + + guard(rwsem_read)(&uctx->fwctl->registration_lock); + if (!uctx->fwctl->ops) + return -ENODEV; + return op->execute(&ucmd); +} + +static int fwctl_fops_open(struct inode *inode, struct file *filp) +{ + struct fwctl_device *fwctl = + container_of(inode->i_cdev, struct fwctl_device, cdev); + int ret; + + guard(rwsem_read)(&fwctl->registration_lock); + if (!fwctl->ops) + return -ENODEV; + + struct fwctl_uctx *uctx __free(kfree) = + kzalloc(fwctl->ops->uctx_size, GFP_KERNEL_ACCOUNT); + if (!uctx) + return -ENOMEM; + + uctx->fwctl = fwctl; + ret = fwctl->ops->open_uctx(uctx); + if (ret) + return ret; + + scoped_guard(mutex, &fwctl->uctx_list_lock) { + list_add_tail(&uctx->uctx_list_entry, &fwctl->uctx_list); + } + + get_device(&fwctl->dev); + filp->private_data = no_free_ptr(uctx); + return 0; +} + +static void fwctl_destroy_uctx(struct fwctl_uctx *uctx) +{ + lockdep_assert_held(&uctx->fwctl->uctx_list_lock); + list_del(&uctx->uctx_list_entry); + uctx->fwctl->ops->close_uctx(uctx); +} + +static int fwctl_fops_release(struct inode *inode, struct file *filp) +{ + struct fwctl_uctx *uctx = filp->private_data; + struct fwctl_device *fwctl = uctx->fwctl; + + scoped_guard(rwsem_read, &fwctl->registration_lock) { + /* + * NULL ops means fwctl_unregister() has already removed the + * driver and destroyed the uctx. + */ + if (fwctl->ops) { + guard(mutex)(&fwctl->uctx_list_lock); + fwctl_destroy_uctx(uctx); + } + } + + kfree(uctx); + fwctl_put(fwctl); + return 0; +} + +static const struct file_operations fwctl_fops = { + .owner = THIS_MODULE, + .open = fwctl_fops_open, + .release = fwctl_fops_release, + .unlocked_ioctl = fwctl_fops_ioctl, +}; + +static void fwctl_device_release(struct device *device) +{ + struct fwctl_device *fwctl = + container_of(device, struct fwctl_device, dev); + + ida_free(&fwctl_ida, fwctl->dev.devt - fwctl_dev); + mutex_destroy(&fwctl->uctx_list_lock); + kfree(fwctl); +} + +static char *fwctl_devnode(const struct device *dev, umode_t *mode) +{ + return kasprintf(GFP_KERNEL, "fwctl/%s", dev_name(dev)); +} + +static struct class fwctl_class = { + .name = "fwctl", + .dev_release = fwctl_device_release, + .devnode = fwctl_devnode, +}; + +static struct fwctl_device * +_alloc_device(struct device *parent, const struct fwctl_ops *ops, size_t size) +{ + struct fwctl_device *fwctl __free(kfree) = kzalloc(size, GFP_KERNEL); + int devnum; + + if (!fwctl) + return NULL; + + devnum = ida_alloc_max(&fwctl_ida, FWCTL_MAX_DEVICES - 1, GFP_KERNEL); + if (devnum < 0) + return NULL; + + fwctl->dev.devt = fwctl_dev + devnum; + fwctl->dev.class = &fwctl_class; + fwctl->dev.parent = parent; + + init_rwsem(&fwctl->registration_lock); + mutex_init(&fwctl->uctx_list_lock); + INIT_LIST_HEAD(&fwctl->uctx_list); + + device_initialize(&fwctl->dev); + return_ptr(fwctl); +} + +/* Drivers use the fwctl_alloc_device() wrapper */ +struct fwctl_device *_fwctl_alloc_device(struct device *parent, + const struct fwctl_ops *ops, + size_t size) +{ + struct fwctl_device *fwctl __free(fwctl) = + _alloc_device(parent, ops, size); + + if (!fwctl) + return NULL; + + cdev_init(&fwctl->cdev, &fwctl_fops); + /* + * The driver module is protected by fwctl_register/unregister(), + * unregister won't complete until we are done with the driver's module. + */ + fwctl->cdev.owner = THIS_MODULE; + + if (dev_set_name(&fwctl->dev, "fwctl%d", fwctl->dev.devt - fwctl_dev)) + return NULL; + + fwctl->ops = ops; + return_ptr(fwctl); +} +EXPORT_SYMBOL_NS_GPL(_fwctl_alloc_device, FWCTL); + +/** + * fwctl_register - Register a new device to the subsystem + * @fwctl: Previously allocated fwctl_device + * + * On return the device is visible through sysfs and /dev, driver ops may be + * called. + */ +int fwctl_register(struct fwctl_device *fwctl) +{ + return cdev_device_add(&fwctl->cdev, &fwctl->dev); +} +EXPORT_SYMBOL_NS_GPL(fwctl_register, FWCTL); + +/** + * fwctl_unregister - Unregister a device from the subsystem + * @fwctl: Previously allocated and registered fwctl_device + * + * Undoes fwctl_register(). On return no driver ops will be called. The + * caller must still call fwctl_put() to free the fwctl. + * + * Unregister will return even if userspace still has file descriptors open. + * This will call ops->close_uctx() on any open FDs and after return no driver + * op will be called. The FDs remain open but all fops will return -ENODEV. + * + * The design of fwctl allows this sort of disassociation of the driver from the + * subsystem primarily by keeping memory allocations owned by the core subsytem. + * The fwctl_device and fwctl_uctx can both be freed without requiring a driver + * callback. This allows the module to remain unlocked while FDs are open. + */ +void fwctl_unregister(struct fwctl_device *fwctl) +{ + struct fwctl_uctx *uctx; + + cdev_device_del(&fwctl->cdev, &fwctl->dev); + + /* Disable and free the driver's resources for any still open FDs. */ + guard(rwsem_write)(&fwctl->registration_lock); + guard(mutex)(&fwctl->uctx_list_lock); + while ((uctx = list_first_entry_or_null(&fwctl->uctx_list, + struct fwctl_uctx, + uctx_list_entry))) + fwctl_destroy_uctx(uctx); + + /* + * The driver module may unload after this returns, the op pointer will + * not be valid. + */ + fwctl->ops = NULL; +} +EXPORT_SYMBOL_NS_GPL(fwctl_unregister, FWCTL); + +static int __init fwctl_init(void) +{ + int ret; + + ret = alloc_chrdev_region(&fwctl_dev, 0, FWCTL_MAX_DEVICES, "fwctl"); + if (ret) + return ret; + + ret = class_register(&fwctl_class); + if (ret) + goto err_chrdev; + return 0; + +err_chrdev: + unregister_chrdev_region(fwctl_dev, FWCTL_MAX_DEVICES); + return ret; +} + +static void __exit fwctl_exit(void) +{ + class_unregister(&fwctl_class); + unregister_chrdev_region(fwctl_dev, FWCTL_MAX_DEVICES); +} + +module_init(fwctl_init); +module_exit(fwctl_exit); +MODULE_DESCRIPTION("fwctl device firmware access framework"); +MODULE_LICENSE("GPL"); diff --git a/drivers/fwctl/ub/Makefile b/drivers/fwctl/ub/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..391aa5f909c55ba70c9f2f0c68a8bb22ba3925ce --- /dev/null +++ b/drivers/fwctl/ub/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0+ +obj-$(CONFIG_FWCTL_UB) += ub_fwctl.o + +ub_fwctl-y += main.o ub_cmd_reg.o ub_common.o ub_cmd.o diff --git a/drivers/fwctl/ub/main.c b/drivers/fwctl/ub/main.c new file mode 100644 index 0000000000000000000000000000000000000000..6b1f619dc0a46a6243f819b2043d68a2d7d22aaa --- /dev/null +++ b/drivers/fwctl/ub/main.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. + */ + +#include +#include +#include +#include + +#include "ub_common.h" +#include "ub_cmd_reg.h" +#include "ub_cmd.h" + +#define MAX_IOCTL_COUNT 1024 +#define TIME_WINDOW_MS 3000 +#define TIME_WINDOW_JIFFIES msecs_to_jiffies(TIME_WINDOW_MS) + +struct ubctl_uctx { + struct fwctl_uctx uctx; +}; + +static int ubctl_open_uctx(struct fwctl_uctx *uctx) +{ + return 0; +} + +static void ubctl_close_uctx(struct fwctl_uctx *uctx) +{ + +} + +static void *ubctl_fw_info(struct fwctl_uctx *uctx, size_t *length) +{ + return NULL; +} + +static int ubctl_legitimacy_rpc(struct ubctl_dev *ucdev, size_t out_len, + enum fwctl_rpc_scope scope) +{ + /* + * Verify if RPC (Remote Procedure Call) requests are valid. + * It determines whether the request is within the allowed time window + * and whether the output length meets the requirements by checking + * the timestamp and output length of the request. + */ + unsigned long current_jiffies = jiffies; + unsigned long earliest_jiffies = current_jiffies - TIME_WINDOW_JIFFIES; + unsigned long record_jiffies = 0; + int kfifo_ret = 0; + + while (kfifo_peek(&ucdev->ioctl_fifo, &record_jiffies) && record_jiffies) { + if (time_after(record_jiffies, earliest_jiffies)) + break; + + kfifo_ret = kfifo_get(&ucdev->ioctl_fifo, &record_jiffies); + if (!kfifo_ret) { + ubctl_err(ucdev, "unexpected events occurred while obtaining data.\n"); + return kfifo_ret; + } + } + + if (kfifo_is_full(&ucdev->ioctl_fifo)) { + ubctl_err(ucdev, "the current number of valid requests exceeds the limit.\n"); + return -EBADMSG; + } + + kfifo_ret = kfifo_put(&ucdev->ioctl_fifo, current_jiffies); + if (!kfifo_ret) { + ubctl_err(ucdev, "unexpected events occurred while writing data.\n"); + return kfifo_ret; + } + + if (out_len < sizeof(struct fwctl_rpc_ub_out)) { + ubctl_dbg(ucdev, "outlen %zu is less than min value %zu.\n", + out_len, sizeof(struct fwctl_rpc_ub_out)); + return -EBADMSG; + } + + if (scope != FWCTL_RPC_CONFIGURATION && + scope != FWCTL_RPC_DEBUG_READ_ONLY) + return -EOPNOTSUPP; + + return 0; +} + +static int ubctl_cmd_err(struct ubctl_dev *ucdev, int ret, struct fwctl_rpc_ub_out *out) +{ + /* Keep rpc_out as contains useful debug info for userspace */ + if (!ret || out->retval) + return 0; + + return ret; +} + +static int ub_cmd_do(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param) +{ + u32 rpc_cmd = query_cmd_param->in->rpc_cmd; + struct ubctl_func_dispatch *ubctl_query_reg = ubctl_get_query_reg_func( + ucdev, rpc_cmd); + struct ubctl_func_dispatch *ubctl_query_func = ubctl_get_query_func( + ucdev, rpc_cmd); + int ret; + + if (ubctl_query_func && ubctl_query_func->execute) { + ret = ubctl_query_func->execute(ucdev, query_cmd_param, + ubctl_query_func); + } else if (ubctl_query_reg && ubctl_query_reg->execute) { + ret = ubctl_query_reg->execute(ucdev, query_cmd_param, + ubctl_query_reg); + } else { + ubctl_err(ucdev, "No corresponding query was found.\n"); + return -EINVAL; + } + + return ubctl_cmd_err(ucdev, ret, query_cmd_param->out); +} + +static void *ubctl_fw_rpc(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope, + void *rpc_in, size_t in_len, size_t *out_len) +{ + struct ubctl_dev *ucdev = container_of(uctx->fwctl, struct ubctl_dev, + fwctl); + u32 opcode = ((struct fwctl_rpc_ub_in *)rpc_in)->rpc_cmd; + struct ubctl_query_cmd_param query_cmd_param; + void *rpc_out; + int ret; + + ubctl_dbg(ucdev, "cmdif: opcode 0x%x inlen %zu outlen %zu\n", + opcode, in_len, *out_len); + + ret = ubctl_legitimacy_rpc(ucdev, *out_len, scope); + if (ret) + return ERR_PTR(ret); + + rpc_out = kvzalloc(*out_len, GFP_KERNEL); + if (!rpc_out) + return ERR_PTR(-ENOMEM); + + query_cmd_param.out = rpc_out; + query_cmd_param.in = rpc_in; + query_cmd_param.out_len = *out_len - offsetof(struct fwctl_rpc_ub_out, data); + query_cmd_param.in_len = in_len; + + ret = ub_cmd_do(ucdev, &query_cmd_param); + + ubctl_dbg(ucdev, "cmdif: opcode 0x%x retval %d\n", opcode, ret); + + if (ret) { + kvfree(rpc_out); + return ERR_PTR(ret); + } + + return rpc_out; +} + +static const struct fwctl_ops ubctl_ops = { + .device_type = FWCTL_DEVICE_TYPE_UB, + .uctx_size = sizeof(struct ubctl_uctx), + .open_uctx = ubctl_open_uctx, + .close_uctx = ubctl_close_uctx, + .info = ubctl_fw_info, + .fw_rpc = ubctl_fw_rpc, +}; + +DEFINE_FREE(ubctl, struct ubctl_dev *, if (_T) fwctl_put(&_T->fwctl)) + +static int ubctl_probe(struct auxiliary_device *adev, + const struct auxiliary_device_id *id) +{ + struct ubctl_dev *ucdev __free(ubctl) = fwctl_alloc_device( + adev->dev.parent, &ubctl_ops, struct ubctl_dev, fwctl); + int ret; + + if (!ucdev) + return -ENOMEM; + + ret = kfifo_alloc(&ucdev->ioctl_fifo, MAX_IOCTL_COUNT, GFP_KERNEL); + if (ret) { + ubctl_err(ucdev, "kfifo alloc device failed, retval = %d.\n", ret); + return -ENOMEM; + } + + ret = fwctl_register(&ucdev->fwctl); + if (ret) { + ubctl_err(ucdev, "fwctl register failed, retval = %d.\n", ret); + kfifo_free(&ucdev->ioctl_fifo); + return ret; + } + + ucdev->adev = adev; + auxiliary_set_drvdata(adev, no_free_ptr(ucdev)); + return 0; +} + +static void ubctl_remove(struct auxiliary_device *adev) +{ + struct ubctl_dev *ucdev = auxiliary_get_drvdata(adev); + + fwctl_unregister(&ucdev->fwctl); + kfifo_free(&ucdev->ioctl_fifo); + fwctl_put(&ucdev->fwctl); +} + +static const struct auxiliary_device_id ubctl_id_table[] = { + { + .name = "ubase.fwctl", + }, + {} +}; +MODULE_DEVICE_TABLE(auxiliary, ubctl_id_table); + +static struct auxiliary_driver ubctl_driver = { + .name = "ub_fwctl", + .probe = ubctl_probe, + .remove = ubctl_remove, + .id_table = ubctl_id_table, +}; + +module_auxiliary_driver(ubctl_driver); + +MODULE_IMPORT_NS(FWCTL); +MODULE_DESCRIPTION("UB fwctl driver"); +MODULE_AUTHOR("HiSilicon Tech. Co., Ltd."); +MODULE_LICENSE("GPL"); diff --git a/drivers/fwctl/ub/ub_cmd.c b/drivers/fwctl/ub/ub_cmd.c new file mode 100644 index 0000000000000000000000000000000000000000..5b3895107b311522212b81be2e37eede67de8565 --- /dev/null +++ b/drivers/fwctl/ub/ub_cmd.c @@ -0,0 +1,1109 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + */ + +#include +#include + +#include "ub_cmdq.h" +#include "ub_cmd.h" + +#define UBCTL_CQE_SIZE 16 +#define UBCTL_SCC_SZ_1M 0x100000 + +static u32 g_ubctl_ummu_reg_addr[] = { + // KCMD + UBCTL_UMMU_SWIF_KCMDQ_DFX_KCMD_STATUS, + UBCTL_UMMU_SWIF_KCMDQ_DFX_KCMD_ERR_STATUS, + // CMD_CTRL + UBCTL_UMMU_SWIF_KCMDQ_DFX_SNP_ERR_CNT, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_0, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_1, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_2, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_3, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_4, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_5, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_6, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_7, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_8, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_9, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_10, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_11, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_12, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_13, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_14, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_15, + UBCTL_UMMU_SWIF_KCMDQ_DFX_SNP_STATUS, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_CTRL_STATUS1, + UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_CTRL_STATUS2, + UBCTL_UMMU_SYNC_TIMEOUT_INFO, + UBCTL_UMMU_DVM_RECEIVE_REQ_CNT, + UBCTL_UMMU_DVM_SEND_REQ_CNT, + UBCTL_UMMU_DVM_REQ_INFO0, + UBCTL_UMMU_DVM_REQ_INFO1, + // UCMD + UBCTL_UMMU_SWIF_UMCMD_DFX0, + UBCTL_UMMU_SWIF_UMCMD_DFX1, + UBCTL_UMMU_SWIF_UMCMD_DFX2, + UBCTL_UMMU_SWIF_UMCMD_DFX3, + UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX0_0, + UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX0_1, + UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX0_2, + UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX0_3, + UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX0_4, + UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX0_5, + UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX0_6, + UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX1, + UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX2, + UBCTL_UMMU_SWIF_UMCMD_CACHE_DFX1, + UBCTL_UMMU_SWIF_UMCMD_CACHE_DFX2, + UBCTL_UMMU_SWIF_UMCMD_CACHE_DFX3, + UBCTL_UMMU_SWIF_UMCMD_CACHE_DFX4, + UBCTL_UMMU_SWIF_UMCMD_CACHE_DFX5, + UBCTL_UMMU_SWIF_UMCMD_CACHE_DFX6, + // EVENT + UBCTL_UMMU_SWIF_EVENTQ_DFX_DROP_CNT, + UBCTL_UMMU_GLB_INT_EN, + UBCTL_UMMU_PMCG_INT_EN, + UBCTL_UMMU_INT_MASK, + UBCTL_UMMU_CTRL1, + UBCTL_UMMU_SPEC_DEF_DFX, + UBCTL_UMMU_TECT_BASE_CFG, + UBCTL_UMMU_ERR_STATUS_0, + UBCTL_UMMU_ROOT_GPF_FAR_L, + UBCTL_UMMU_ROOT_GPF_FAR_H, + UBCTL_UMMU_EVENT_QUE_PI, + UBCTL_UMMU_EVENT_QUE_CI, + // UBIF + UBCTL_UMMU_UBIF_DFX0, + UBCTL_UMMU_UBIF_DFX1, + UBCTL_UMMU_UBIF_DSTEID_DFX, + UBCTL_UMMU_UBIF_SYNC_DFX, + UBCTL_UMMU_UBIF_KV_CACHE_NS_NSE_MISMATCH_DFX0, + UBCTL_UMMU_UBIF_KV_CACHE_NS_NSE_MISMATCH_DFX1, + UBCTL_UMMU_UBIF_KV_CACHE_NS_NSE_MISMATCH_DFX2, + UBCTL_UMMU_UBIF_KV_CACHE_NS_NSE_MISMATCH_DFX3, + UBCTL_UMMU_UBIF_KV_CACHE_NS_NSE_MISMATCH_DFX4, + // TBU + UBCTL_UMMU_TBU_TLB_LKUP_PROC, + UBCTL_UMMU_TBU_TLB_STAT, + UBCTL_UMMU_TBU_TLB_FAULT_CNT, + UBCTL_UMMU_TBU_PLB_LKUP_PROC, + UBCTL_UMMU_TBU_PLB_STAT, + UBCTL_UMMU_TBU_PLB_FAULT_CNT, + UBCTL_UMMU_TBU_INVLD_MG_INFO, + UBCTL_UMMU_TBU_RAB_STAT, + UBCTL_UMMU_TBU_CNT, + UBCTL_UMMU_DFX_TBU_PERM_ERR_CNT, + UBCTL_UMMU_TBU_DFX0, + UBCTL_UMMU_TBU_DFX1, + UBCTL_UMMU_TBU_RAB_ENTRY_INFO_0_7_15, + // TCU + UBCTL_UMMU_TCU_PTW_QUEUE_STAT_0_47, + UBCTL_UMMU_TCU_PPTW_QUEUE_STAT_0_39, + // CFG + UBCTL_UMMU_DFX_ECC_MONITOR_0, + UBCTL_UMMU_DFX_ECC_MONITOR_1, + UBCTL_UMMU_CFG_DFX_CFGBUS_STATUS, + // GPC + UBCTL_UMMU_GPC_QUEUE_STAT_0_15, + // SKY + UBCTL_UMMU_SKY_QUEUE_STAT3_SP_0_63, + // MCMD + UBCTL_UMMU_MCMD_QUE_PI_0, + UBCTL_UMMU_MCMD_QUE_PI_1, + UBCTL_UMMU_MCMD_QUE_PI_2, + UBCTL_UMMU_MCMD_QUE_PI_3, + UBCTL_UMMU_MCMD_QUE_PI_4, + UBCTL_UMMU_MCMD_QUE_PI_5, + UBCTL_UMMU_MCMD_QUE_PI_6, + UBCTL_UMMU_MCMD_QUE_PI_7, + UBCTL_UMMU_MCMD_QUE_CI_0, + UBCTL_UMMU_MCMD_QUE_CI_1, + UBCTL_UMMU_MCMD_QUE_CI_2, + UBCTL_UMMU_MCMD_QUE_CI_3, + UBCTL_UMMU_MCMD_QUE_CI_4, + UBCTL_UMMU_MCMD_QUE_CI_5, + UBCTL_UMMU_MCMD_QUE_CI_6, + UBCTL_UMMU_MCMD_QUE_CI_7, + // UMMU_EN + UBCTL_UMMU_CTRL0, + // OTHER + UBCTL_UMMU_SYNC_TIMEOUT_OPEN, +}; + +struct ubctl_ummu_relation { + u32 reg_addr; + u32 reg_config_addr; + u32 reg_count; +}; + +struct ubctl_query_trace { + u32 port_id; + u32 index; + u32 cur_count; + u32 total_count; + u32 data[]; +}; + +struct ubctl_scc_data { + u32 phy_addr_low; + u32 phy_addr_high; + u32 data_size; + u32 rsv[3]; +}; + +struct ubctl_msgq_to_user { + u32 sq_pi; + u32 sq_ci; + u32 sq_dep; + u32 sq_status; + u32 sq_int_mask; + u32 sq_int_status; + u32 sq_int_ro; + + u32 rq_pi; + u32 rq_ci; + u32 rq_dep; + u32 rq_entry_block_size; + u32 rq_status; + + u32 cq_pi; + u32 cq_ci; + u32 cq_dep; + u32 cq_status; + u32 cq_int_mask; + u32 cq_int_status; + u32 cq_int_ro; + + u32 rsvd[5]; +}; + +struct ubctl_msgq { + u32 sq_base_addr_low; + u32 sq_base_addr_high; + u32 sq_pi; + u32 sq_ci; + u32 sq_dep; + u32 sq_status; + u32 sq_int_mask; + u32 sq_int_status; + u32 sq_int_ro; + + u32 rq_base_addr_low; + u32 rq_base_addr_high; + u32 rq_pi; + u32 rq_ci; + u32 rq_dep; + u32 rq_entry_block_size; + u32 rq_status; + + u32 cq_base_addr_low; + u32 cq_base_addr_high; + u32 cq_pi; + u32 cq_ci; + u32 cq_dep; + u32 cq_status; + u32 cq_int_mask; + u32 cq_int_status; + u32 cq_int_ro; + + u32 resv[5]; +}; + +struct ubctl_msgq_phy_addr { + u64 sq_entry_phy_addr; + u64 cq_entry_phy_addr; +}; + +static int ubctl_trace_data_deal(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_cmd *cmd, u32 out_len, u32 offset) +{ +#define UBCTL_TRACE_SIZE 4U +#define UBCTL_TOTAL_CNT_MAX 64U + + struct fwctl_rpc_ub_out *trace_out = query_cmd_param->out; + struct ubctl_query_trace *trace_info = cmd->out_data; + u32 trace_max_len = query_cmd_param->out_len; + u32 pos_index = offset * UBCTL_TRACE_SIZE; + + if ((trace_info->total_count > UBCTL_TOTAL_CNT_MAX) || + (trace_info->total_count * UBCTL_TRACE_SIZE >= trace_max_len) || + (pos_index >= trace_max_len || cmd->out_len < sizeof(struct ubctl_query_trace))) { + ubctl_err(ucdev, "cmd out data length is error.\n"); + return -EINVAL; + } + + if (pos_index == 0) + memcpy(trace_out->data, cmd->out_data, cmd->out_len); + else + memcpy((u32 *)(trace_out->data) + pos_index, trace_info->data, + cmd->out_len - sizeof(struct ubctl_query_trace)); + + trace_out->data_size = query_cmd_param->out_len; + return 0; +} + +static int ubctl_send_deal_trace(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_query_cmd_dp *cmd_data, u32 offset) +{ + u32 out_len = UBCTL_DL_TRACE_LEN; + struct ubctl_cmd cmd = {}; + int ret = 0; + + if (!cmd_data->query_func->data_deal) { + ubctl_err(ucdev, "ubctl data deal func is null.\n"); + return -EINVAL; + } + + cmd.op_code = UBCTL_QUERY_DL_TRACE_DFX; + + ret = ubctl_fill_cmd(&cmd, cmd_data->cmd_in, cmd_data->cmd_out, + out_len, UBCTL_READ); + if (ret) { + ubctl_err(ucdev, "ubctl fill cmd failed.\n"); + return ret; + } + + ret = ubctl_ubase_cmd_send(ucdev->adev, &cmd); + if (ret) { + ubctl_err(ucdev, "ubctl ubase cmd send failed, ret = %d.\n", ret); + return -EINVAL; + } + + ret = cmd_data->query_func->data_deal(ucdev, query_cmd_param, &cmd, + out_len, offset); + if (ret) + ubctl_err(ucdev, "ubctl data deal failed, ret = %d.\n", ret); + + return ret; +} + +static int ubctl_query_dl_trace_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct fwctl_pkt_in_port *pkt_in = (struct fwctl_pkt_in_port *)query_cmd_param->in->data; + u32 trace_index = 0, offset = 0, expect_total = 0, out_len = UBCTL_DL_TRACE_LEN, tmp_sum; + struct ubctl_query_cmd_dp cmd_dp = {}; + int ret = 0; + + if (query_cmd_param->in->data_size != sizeof(struct fwctl_pkt_in_port)) { + ubctl_err(ucdev, "user data of trace is invalid.\n"); + return -EINVAL; + } + + while (1) { + struct ubctl_query_trace *cmd_in __free(kvfree) = kvzalloc(out_len, GFP_KERNEL); + if (!cmd_in) + return -ENOMEM; + + struct ubctl_query_trace *cmd_out __free(kvfree) = kvzalloc(out_len, GFP_KERNEL); + if (!cmd_out) + return -ENOMEM; + + cmd_in->index = trace_index; + cmd_in->port_id = pkt_in->port_id; + + cmd_dp = (struct ubctl_query_cmd_dp) { + .cmd_in = cmd_in, + .cmd_out = cmd_out, + .query_func = query_func, + }; + + ret = ubctl_send_deal_trace(ucdev, query_cmd_param, + &cmd_dp, offset); + if (ret) + return ret; + + offset = cmd_out->cur_count + 1; + trace_index = cmd_out->cur_count; + tmp_sum = cmd_out->cur_count + cmd_in->index; + + if ((tmp_sum <= expect_total) || (tmp_sum > cmd_out->total_count)) { + ubctl_err(ucdev, "software data of trace is invalid.\n"); + return -EINVAL; + } + + if (tmp_sum == cmd_out->total_count) + break; + + expect_total = tmp_sum; + } + + return ret; +} + +static int ubctl_scc_data_deal(struct ubctl_dev *ucdev, u32 index, + struct fwctl_rpc_ub_out *out, + struct ubctl_scc_data *scc) +{ +#define UBCTL_SCC_OUT_LEN ((UBCTL_SCC_SZ_1M) / (sizeof(u32))) +#define UBCTL_SCC_INDEX_MAX_NUM 1 + + u32 scc_data_len = scc->data_size / sizeof(u32); + u32 data_len = out->data_size / sizeof(u32); + u32 offset = index * UBCTL_SCC_OUT_LEN; + u32 *scc_data = out->data; + void __iomem *vir_addr; + u64 phy_addr; + u32 i, j; + + if (index > UBCTL_SCC_INDEX_MAX_NUM) { + ubctl_err(ucdev, "scc index is invalid, index = %u.\n", index); + return -EINVAL; + } + + phy_addr = UBCTL_GET_PHY_ADDR(scc->phy_addr_high, scc->phy_addr_low); + + vir_addr = ioremap(phy_addr, scc->data_size); + if (!vir_addr) { + ubctl_err(ucdev, "addr ioremap failed.\n"); + return -EFAULT; + } + + for (i = offset, j = 0; i < scc_data_len && j < data_len; i++, j++) + scc_data[j] = readl(vir_addr + i * sizeof(u32)); + + iounmap(vir_addr); + return 0; +} + +static int ubctl_scc_data_len_check(struct ubctl_dev *ucdev, u32 out_len, + u32 data_size, u32 scc_len) +{ +#define UBCTL_SCC_CACHE 0x200000 + + if (data_size != UBCTL_SCC_CACHE) { + ubctl_err(ucdev, "scc data size is not equal to 2M, data size = %u.\n", + data_size); + return -EINVAL; + } + + if (out_len != scc_len) { + ubctl_err(ucdev, "scc out len is invalid, out len = %u.\n", + out_len); + return -EINVAL; + } + + return 0; +} + +static int ubctl_scc_version_deal(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_cmd *cmd, u32 out_len, u32 offset) +{ +#define UBCTL_SCC_VERSION_SZ 24 + + struct fwctl_pkt_in_index *pkt_in = NULL; + struct ubctl_scc_data *scc = NULL; + int ret = 0; + + if (query_cmd_param->in->data_size != sizeof(struct fwctl_pkt_in_index)) { + ubctl_err(ucdev, "user data of scc version is invalid.\n"); + return -EINVAL; + } + pkt_in = (struct fwctl_pkt_in_index *)query_cmd_param->in->data; + scc = (struct ubctl_scc_data *)cmd->out_data; + + ret = ubctl_scc_data_len_check(ucdev, query_cmd_param->out_len, + scc->data_size, UBCTL_SCC_VERSION_SZ); + if (ret) { + ubctl_err(ucdev, "scc version data len check failed, ret = %d.\n", ret); + return -EINVAL; + } + + query_cmd_param->out->data_size = query_cmd_param->out_len; + scc->data_size = sizeof(u32); + + return ubctl_scc_data_deal(ucdev, pkt_in->index, query_cmd_param->out, scc); +} + +static int ubctl_scc_log_deal(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_cmd *cmd, u32 out_len, u32 offset) +{ + struct fwctl_pkt_in_index *pkt_in = (struct fwctl_pkt_in_index *)query_cmd_param->in->data; + struct ubctl_scc_data *scc = (struct ubctl_scc_data *)cmd->out_data; + int ret = 0; + + if (query_cmd_param->in->data_size != sizeof(*pkt_in)) { + ubctl_err(ucdev, "user data of scc log is invalid.\n"); + return -EINVAL; + } + + ret = ubctl_scc_data_len_check(ucdev, query_cmd_param->out_len, + scc->data_size, UBCTL_SCC_SZ_1M); + if (ret) { + ubctl_err(ucdev, "scc log data len check failed, ret = %d.\n", ret); + return -EINVAL; + } + + query_cmd_param->out->data_size = query_cmd_param->out_len; + + return ubctl_scc_data_deal(ucdev, pkt_in->index, query_cmd_param->out, scc); +} + +static int ubctl_query_scc_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_SCC_DFX, UBCTL_SCC_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_port_infos(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func, + u32 port_bitmap) +{ +#define UBCTL_U32_BIT_NUM 32U + + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_DL_LINK_STATUS_DFX, UBCTL_DL_LINK_STATUS_LEN, UBCTL_READ, NULL, 0 }, + }; + u32 iodie_len = sizeof(struct fwctl_rpc_ub_out) + UBCTL_IO_DIE_INFO_LEN; + u32 out_data_offset = UBCTL_IO_DIE_INFO_LEN / sizeof(u32); + struct fwctl_rpc_ub_out *out = query_cmd_param->out; + u32 out_mem_size = query_cmd_param->out_len; + u32 *pkt_in = query_cmd_param->in->data; + u32 data_size = out->data_size; + int port_num = 0; + int ret = 0; + u32 i; + + if (port_bitmap == 0) + return ret; + + struct fwctl_rpc_ub_out *out_temp __free(kvfree) = kvzalloc(iodie_len, GFP_KERNEL); + if (!out_temp) + return -ENOMEM; + + query_cmd_param->out = out_temp; + + for (i = 0; i < UBCTL_U32_BIT_NUM; i++) { + if (!(port_bitmap & (1UL << i))) + continue; + out_temp->data_size = 0; + *pkt_in = i; + ret = ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); + if (ret != 0) + break; + + if ((out_temp->data_size + out_data_offset * sizeof(u32)) > out_mem_size) { + ubctl_err(ucdev, "port info size = %u, total size = %u, offset size = %lu.\n", + out_temp->data_size, out_mem_size, + out_data_offset * sizeof(u32)); + ret = -ENOMEM; + break; + } + + memcpy(&out->data[out_data_offset], out_temp->data, out_temp->data_size); + data_size += out_temp->data_size; + out_data_offset += UBCTL_DL_LINK_STATUS_LEN / sizeof(u32); + port_num++; + } + + query_cmd_param->out = out; + out->data_size = data_size; + out->data[0] = port_num; + + return ret; +} + +static int ubctl_query_iodie_info_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_PORT_NUM_DFX, UBCTL_IO_DIE_INFO_LEN, UBCTL_READ, NULL, 0 }, + }; + u32 port_bitmap; + int ret; + + ret = ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); + if (ret != 0) + return ret; + + if (query_cmd_param->out->data_size < sizeof(u32)) + return -ENOMEM; + port_bitmap = *query_cmd_param->out->data; + + return ubctl_query_port_infos(ucdev, query_cmd_param, query_func, port_bitmap); +} + +static int ubctl_msgq_que_data_deal(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_cmd *cmd, u32 out_len, u32 offset) +{ + struct ubctl_msgq *msgq_que_info = (struct ubctl_msgq *)cmd->out_data; + struct ubctl_msgq_to_user *user_msgq_info = NULL; + u32 msgq_que_size = query_cmd_param->out_len; + + if (cmd->out_len != out_len || + msgq_que_size != sizeof(struct ubctl_msgq_to_user)) + return -EINVAL; + + user_msgq_info = (struct ubctl_msgq_to_user *)(query_cmd_param->out->data); + + user_msgq_info->sq_pi = msgq_que_info->sq_pi; + user_msgq_info->sq_ci = msgq_que_info->sq_ci; + user_msgq_info->sq_dep = msgq_que_info->sq_dep; + user_msgq_info->sq_status = msgq_que_info->sq_status; + user_msgq_info->sq_int_mask = msgq_que_info->sq_int_mask; + user_msgq_info->sq_int_status = msgq_que_info->sq_int_status; + user_msgq_info->sq_int_ro = msgq_que_info->sq_int_ro; + + user_msgq_info->rq_pi = msgq_que_info->rq_pi; + user_msgq_info->rq_ci = msgq_que_info->rq_ci; + user_msgq_info->rq_dep = msgq_que_info->rq_dep; + user_msgq_info->rq_entry_block_size = msgq_que_info->rq_entry_block_size; + user_msgq_info->rq_status = msgq_que_info->rq_status; + + user_msgq_info->cq_pi = msgq_que_info->cq_pi; + user_msgq_info->cq_ci = msgq_que_info->cq_ci; + user_msgq_info->cq_dep = msgq_que_info->cq_dep; + user_msgq_info->cq_status = msgq_que_info->cq_status; + user_msgq_info->cq_int_mask = msgq_que_info->cq_int_mask; + user_msgq_info->cq_int_status = msgq_que_info->cq_int_status; + user_msgq_info->cq_int_ro = msgq_que_info->cq_int_ro; + + query_cmd_param->out->data_size = msgq_que_size; + + return 0; +} + +static int ubctl_msgq_is_dump(void __iomem *entry_addr) +{ +#define UBCTL_MSGQ_PROTOCOL_OPCODE 5 +#define UBCTL_MSGQ_OPCODE_START 9 +#define UBCTL_MSGQ_OPCODE_END 11 + + u32 first_data = readl(entry_addr); + u32 protocol_op_code = 0; + u32 task_type = 0; + + protocol_op_code = UBCTL_EXTRACT_BITS(first_data, + UBCTL_MSGQ_OPCODE_START, + UBCTL_MSGQ_OPCODE_END); + task_type = UBCTL_EXTRACT_BITS(first_data, 0, 1); + if (task_type == 0 && protocol_op_code == UBCTL_MSGQ_PROTOCOL_OPCODE) + return -EINVAL; + + return 0; +} + +static int ubctl_msgq_entry_move_data(struct ubctl_query_cmd_param *query_cmd_param, + u32 offset, u32 block_size, + void __iomem *entry_addr) +{ + u32 msgq_entry_data_size = block_size + offset * sizeof(u32); + u32 *data_offset = query_cmd_param->out->data + offset; + u32 i; + + if (msgq_entry_data_size > query_cmd_param->out_len) + return -EINVAL; + + for (i = 0; i < block_size / sizeof(u32); i++) + data_offset[i] = readl(entry_addr + i); + + return 0; +} + +static int ubctl_msgq_check_index(struct ubctl_dev *ucdev, u32 entry_index, + struct ubctl_msgq *entry_info) +{ + if (entry_index >= entry_info->sq_dep || + entry_index >= entry_info->cq_dep) { + ubctl_err(ucdev, "index is illegal, index = %u.\n", entry_index); + return -EINVAL; + } + + return 0; +} + +static int ubctl_msgq_all_get_phy_addr(struct ubctl_dev *ucdev, u32 entry_index, + struct ubctl_msgq_phy_addr *entry_phy_addr, + struct ubctl_msgq *entry_info) +{ +#define UBCTL_SQE_SIZE 16 + + u64 base_addr; + int ret; + + ret = ubctl_msgq_check_index(ucdev, entry_index, entry_info); + if (ret) + return ret; + + base_addr = UBCTL_GET_PHY_ADDR(entry_info->sq_base_addr_high, + entry_info->sq_base_addr_low); + if (!base_addr) { + ubctl_err(ucdev, "sqe msgq not initialized.\n"); + return -EINVAL; + } + + entry_phy_addr->sq_entry_phy_addr = base_addr + + entry_index * UBCTL_SQE_SIZE; + + base_addr = UBCTL_GET_PHY_ADDR(entry_info->cq_base_addr_high, + entry_info->cq_base_addr_low); + if (!base_addr) { + ubctl_err(ucdev, "cqe msgq not initialized.\n"); + return -EINVAL; + } + + entry_phy_addr->cq_entry_phy_addr = base_addr + + entry_index * UBCTL_CQE_SIZE; + + return 0; +} + +static int ubctl_msgq_sq_entry_data_deal(struct ubctl_dev *ucdev, + u64 sq_entry_phy_addr, + struct ubctl_query_cmd_param *query_cmd_param) +{ +#define UBCTL_SQE_TO_USER_SIZE 8 + + void __iomem *sq_entry_addr; + int ret = 0; + + sq_entry_addr = memremap(sq_entry_phy_addr, UBCTL_SQE_TO_USER_SIZE, MEMREMAP_WB); + if (!sq_entry_addr) + return -EFAULT; + + ret = ubctl_msgq_is_dump(sq_entry_addr); + if (ret) { + ubctl_err(ucdev, "this entry cannot be dumped, sqe is SPDM verified msg.\n"); + goto err_exec; + } + + ret = ubctl_msgq_entry_move_data(query_cmd_param, 0, + UBCTL_SQE_TO_USER_SIZE, sq_entry_addr); + if (ret) + ubctl_err(ucdev, "move sqe data failed.\n"); + +err_exec: + memunmap(sq_entry_addr); + return ret; +} + +static int ubctl_msgq_cq_entry_data_deal(struct ubctl_dev *ucdev, + u64 cq_entry_phy_addr, + struct ubctl_query_cmd_param *query_cmd_param) +{ +#define UBCTL_CQE_OFFSET 2 + + void __iomem *cq_entry_addr; + int ret = 0; + + cq_entry_addr = memremap(cq_entry_phy_addr, UBCTL_CQE_SIZE, MEMREMAP_WB); + if (!cq_entry_addr) + return -EFAULT; + + ret = ubctl_msgq_is_dump(cq_entry_addr); + if (ret) { + ubctl_err(ucdev, "this entry cannot be dumped, cqe is SPDM verified msg.\n"); + goto err_exec; + } + + ret = ubctl_msgq_entry_move_data(query_cmd_param, UBCTL_CQE_OFFSET, + UBCTL_CQE_SIZE, cq_entry_addr); + if (ret) + ubctl_err(ucdev, "move cqe data failed.\n"); + +err_exec: + memunmap(cq_entry_addr); + return ret; +} + +static int ubctl_msgq_entry_data_deal(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_cmd *cmd, u32 out_len, u32 offset) +{ + struct ubctl_msgq *entry_info = (struct ubctl_msgq *)cmd->out_data; + u32 msgq_entry_max_len = query_cmd_param->out_len; + struct ubctl_msgq_phy_addr entry_phy_addr = {}; + u32 entry_index = 0; + int ret = 0; + + if (query_cmd_param->in->data_size != sizeof(struct fwctl_pkt_in_index)) { + ubctl_err(ucdev, "user data of msgq is invalid.\n"); + return -EINVAL; + } + entry_index = ((struct fwctl_pkt_in_index *)query_cmd_param->in->data)->index; + + ret = ubctl_msgq_all_get_phy_addr(ucdev, entry_index, &entry_phy_addr, + entry_info); + if (ret) + return ret; + + ret = ubctl_msgq_sq_entry_data_deal(ucdev, + entry_phy_addr.sq_entry_phy_addr, + query_cmd_param); + if (ret) + return ret; + + ret = ubctl_msgq_cq_entry_data_deal(ucdev, + entry_phy_addr.cq_entry_phy_addr, + query_cmd_param); + if (ret) + return ret; + + query_cmd_param->out->data_size = msgq_entry_max_len; + + return ret; +} + +static int ubctl_query_msgq_que_stats_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_MSGQ_DFX, UBCTL_MSGQ_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int compare_resources(const void *a, const void *b) +{ + const struct resource *ra = *(const struct resource **)a; + const struct resource *rb = *(const struct resource **)b; + + if (ra->start < rb->start) + return -1; + if (ra->start > rb->start) + return 1; + return 0; +} + +static struct resource *ubctl_find_and_sort_resources(struct ubctl_dev *ucdev, + struct resource *root, + const char *name_substr, + u32 ummu_id) +{ +#define UBCL_MAX_UMMU_NUM 32U + + struct resource *entry_arr[UBCL_MAX_UMMU_NUM] = {}; + struct resource *p; + u32 count = 0; + + /* + * To traverse the UMMU memory subtree, only need to traverse the child + * subtree of the root node. + */ + for (p = root->child; p; p = p->sibling) { + if (!p->name || !strstr(p->name, name_substr)) + continue; + if (count >= UBCL_MAX_UMMU_NUM) { + ubctl_err(ucdev, "ummu resources is more than max num = %u.\n", + UBCL_MAX_UMMU_NUM); + return NULL; + } + entry_arr[count] = p; + count++; + } + + if (ummu_id >= count) { + ubctl_err(ucdev, "ummuid = %u out of range, current count = %u\n", + ummu_id, count); + return NULL; + } + + sort(entry_arr, count, sizeof(struct resource *), compare_resources, NULL); + + return entry_arr[ummu_id]; +} + +static inline u32 ubctl_ummu_get_register_offset(u32 index) +{ + return g_ubctl_ummu_reg_addr[index] - UBCTL_UMMU_REGISTER_BASE; +} + +static inline u32 ubctl_ummu_get_reg_count(void) +{ +#define UBCTL_UMMU_REPEAT_REG_TYPE_COUNT 5U + + return ARRAY_SIZE(g_ubctl_ummu_reg_addr) + UBCTL_UMMU_GPC_QUEUE_COUNT + + UBCTL_UMMU_SKY_QUEUE_COUNT + UBCTL_UMMU_TCU_PTW_QUEUE_COUNT + + UBCTL_UMMU_TCU_PPTW_QUEUE_COUNT + UBCTL_UMMU_ENTRY_NUM * + UBCTL_UMMU_BANK_NUM - UBCTL_UMMU_REPEAT_REG_TYPE_COUNT; +} + +struct ubctl_reg_pro_cmd { + struct ubctl_dev *ucdev; + u32 reg_index; + void __iomem *map_addr; + u32 *ummu_data; + u32 map_length; + u32 *index_offset; +}; + +static int ubctl_ummu_normal_read(struct ubctl_reg_pro_cmd *cmd) +{ + u32 ummu_reg_cnt = ubctl_ummu_get_reg_count(); + u32 reg_addr_offset; + + reg_addr_offset = ubctl_ummu_get_register_offset(cmd->reg_index); + if ((reg_addr_offset >= cmd->map_length) || (*cmd->index_offset >= ummu_reg_cnt)) { + ubctl_err(cmd->ucdev, "ummu reg offset is bigger than map length, index=%u, reg offset=%u, map length=%u.\n", + *cmd->index_offset, reg_addr_offset, cmd->map_length); + return -EFAULT; + } + cmd->ummu_data[*cmd->index_offset] = readl(cmd->map_addr + reg_addr_offset); + (*cmd->index_offset)++; + + return 0; +} + +static int ubctl_ummu_process_repeat_reg(struct ubctl_reg_pro_cmd *cmd) +{ + static struct ubctl_ummu_relation ummu_relation[] = { + { UBCTL_UMMU_GPC_QUEUE_STAT_0_15, UBCTL_UMMU_GPC_QUEUE_POINTER, + UBCTL_UMMU_GPC_QUEUE_COUNT }, + { UBCTL_UMMU_SKY_QUEUE_STAT3_SP_0_63, UBCTL_UMMU_SKY_QUEUE_POINTER_SP, + UBCTL_UMMU_SKY_QUEUE_COUNT }, + { UBCTL_UMMU_TCU_PTW_QUEUE_STAT_0_47, UBCTL_UMMU_TCU_PTW_QUEUE_POINTER, + UBCTL_UMMU_TCU_PTW_QUEUE_COUNT }, + { UBCTL_UMMU_TCU_PPTW_QUEUE_STAT_0_39, UBCTL_UMMU_TCU_PPTW_QUEUE_POINTER, + UBCTL_UMMU_TCU_PPTW_QUEUE_COUNT } + }; + + u32 read_reg_offset, set_reg_offset, write_count, i, j; + u32 ummu_reg_cnt = ubctl_ummu_get_reg_count(); + + for (i = 0; i < ARRAY_SIZE(ummu_relation); i++) { + if (ummu_relation[i].reg_addr != g_ubctl_ummu_reg_addr[cmd->reg_index]) + continue; + write_count = ummu_relation[i].reg_count; + set_reg_offset = ummu_relation[i].reg_config_addr - + UBCTL_UMMU_REGISTER_BASE; + read_reg_offset = ummu_relation[i].reg_addr - + UBCTL_UMMU_REGISTER_BASE; + if ((set_reg_offset >= cmd->map_length) || + (read_reg_offset >= cmd->map_length)) { + ubctl_err(cmd->ucdev, "ummu set or read reg offset is bigger than map length, set offset=%u, read offset=%u, map length=%u.\n", + set_reg_offset, read_reg_offset, cmd->map_length); + return -EFAULT; + } + + for (j = 0; j < write_count; j++, (*cmd->index_offset)++) { + writel(j, cmd->map_addr + set_reg_offset); + if (*cmd->index_offset >= ummu_reg_cnt) { + ubctl_err(cmd->ucdev, "index offset is bigger than ummu reg count, index offset=%u, ummu reg count=%u.\n", + *cmd->index_offset, ummu_reg_cnt); + return -EFAULT; + } + cmd->ummu_data[*cmd->index_offset] = readl(cmd->map_addr + + read_reg_offset); + } + return 0; + } + + return ubctl_ummu_normal_read(cmd); +} + +static int ubctl_ummu_process_reg(struct ubctl_reg_pro_cmd *cmd) +{ +#define UBCTL_TBU_MASK 0xFFFFFC00U +#define UBCTL_BANK_OFFSET 6 + + u32 read_reg_offset, set_reg_offset, origin_value, value, i, j; + u32 ummu_reg_cnt = ubctl_ummu_get_reg_count(); + + if (g_ubctl_ummu_reg_addr[cmd->reg_index] != UBCTL_UMMU_TBU_RAB_ENTRY_INFO_0_7_15) + return ubctl_ummu_process_repeat_reg(cmd); + + set_reg_offset = UBCTL_UMMU_TBU_RAB_FUNC_EN - UBCTL_UMMU_REGISTER_BASE; + read_reg_offset = UBCTL_UMMU_TBU_RAB_ENTRY_INFO_0_7_15 - + UBCTL_UMMU_REGISTER_BASE; + if ((set_reg_offset >= cmd->map_length) || + (read_reg_offset >= cmd->map_length)) { + ubctl_err(cmd->ucdev, "ummu set or read reg offset is bigger than map length, set offset=%u, read offset=%u, map length=%u.\n", + set_reg_offset, read_reg_offset, cmd->map_length); + return -EFAULT; + } + + origin_value = readl(cmd->map_addr + set_reg_offset); + origin_value &= UBCTL_TBU_MASK; + for (i = 0; i < UBCTL_UMMU_BANK_NUM; i++) { + for (j = 0; j < UBCTL_UMMU_ENTRY_NUM; j++, (*cmd->index_offset)++) { + value = (i << UBCTL_BANK_OFFSET) | j | origin_value; + writel(value, cmd->map_addr + set_reg_offset); + if (*cmd->index_offset >= ummu_reg_cnt) { + ubctl_err(cmd->ucdev, "index offset is bigger than ummu reg count, index offset=%u, ummu reg count=%u.\n", + *cmd->index_offset, ummu_reg_cnt); + return -EFAULT; + } + cmd->ummu_data[*cmd->index_offset] = readl(cmd->map_addr + + read_reg_offset); + } + } + return 0; +} + +static int ubctl_ummu_copy_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + void __iomem *map_addr, u32 map_length) +{ + u32 ummu_array_cnt = ARRAY_SIZE(g_ubctl_ummu_reg_addr); + u32 ummu_reg_cnt = ubctl_ummu_get_reg_count(); + u32 *ummu_data = query_cmd_param->out->data; + u32 index_offset = 0; + int ret; + u32 i; + + struct ubctl_reg_pro_cmd reg_pro_cmd = { + .ucdev = ucdev, + .reg_index = 0, + .map_addr = map_addr, + .ummu_data = ummu_data, + .map_length = map_length, + .index_offset = &index_offset, + }; + + if (ummu_reg_cnt * sizeof(u32) > query_cmd_param->out_len) { + ubctl_err(ucdev, "ummu reg size is big than out len, reg sie=%lu, out len=%lu.\n", + ummu_reg_cnt * sizeof(u32), query_cmd_param->out_len); + return -EINVAL; + } + + for (i = 0; i < ummu_array_cnt; i++) { + reg_pro_cmd.reg_index = i; + ret = ubctl_ummu_process_reg(®_pro_cmd); + if (ret) { + ubctl_err(ucdev, "ummu process reg failed, ret=%d.\n", ret); + return ret; + } + } + query_cmd_param->out->data_size = ummu_reg_cnt * sizeof(u32); + + return 0; +} + +static int ubctl_ummu_proc_all_data(struct ubctl_dev *ucdev, struct resource *res, + struct ubctl_query_cmd_param *query_cmd_param) +{ + u32 map_length = UBCTL_UMMU_REGISTER_MAX_ADDR - UBCTL_UMMU_REGISTER_BASE; + void __iomem *vaddr; + int ret; + + vaddr = ioremap(res->start + UBCTL_UMMU_REGISTER_BASE, map_length); + if (!vaddr) { + ubctl_err(ucdev, "ioremap ummu reg base failed, map length = %u.\n", + map_length); + return -ENOMEM; + } + ret = ubctl_ummu_copy_data(ucdev, query_cmd_param, vaddr, map_length); + iounmap(vaddr); + + return ret; +} + +static int ubctl_ummu_proc_sync_data(struct resource *res, + struct ubctl_query_cmd_param *query_cmd_param, + struct fwctl_pkt_in_ummuid_value *ummu_data, + bool is_query) +{ + u32 *out_data = query_cmd_param->out->data; + u32 map_length = sizeof(u32); + void __iomem *vaddr; + + if (sizeof(u32) > query_cmd_param->out_len) + return -EINVAL; + + vaddr = ioremap(res->start + UBCTL_UMMU_SYNC_TIMEOUT_OPEN, map_length); + if (!vaddr) + return -ENOMEM; + + if (is_query) { + *out_data = readl(vaddr); + } else { + *out_data = ummu_data->value; + writel(*out_data, vaddr); + } + + query_cmd_param->out->data_size = sizeof(u32); + iounmap(vaddr); + + return 0; +} + +static int ubctl_ummu_process_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ +#define UMMU_NAME_STR "ummu." + + struct fwctl_pkt_in_ummuid_value *ummu_data; + struct resource *root = &iomem_resource; + struct resource *res; + + if (query_cmd_param->in->data_size != sizeof(*ummu_data)) { + ubctl_err(ucdev, "invalid ummuid value size = %u.\n", + query_cmd_param->in->data_size); + return -EINVAL; + } + + ummu_data = (struct fwctl_pkt_in_ummuid_value *)(query_cmd_param->in->data); + res = ubctl_find_and_sort_resources(ucdev, root, UMMU_NAME_STR, + ummu_data->ummu_id); + if (!res) + return -EINVAL; + + if (query_func->rpc_cmd == UTOOL_CMD_QUERY_UMMU_ALL) + return ubctl_ummu_proc_all_data(ucdev, res, query_cmd_param); + if (query_func->rpc_cmd == UTOOL_CMD_QUERY_UMMU_SYNC) + return ubctl_ummu_proc_sync_data(res, query_cmd_param, ummu_data, true); + if (query_func->rpc_cmd == UTOOL_CMD_CONFIG_UMMU_SYNC) + return ubctl_ummu_proc_sync_data(res, query_cmd_param, ummu_data, false); + + return -EINVAL; +} + +static struct ubctl_func_dispatch g_ubctl_query_func[] = { + { UTOOL_CMD_QUERY_DL_LINK_TRACE, ubctl_query_dl_trace_data, + ubctl_trace_data_deal }, + + { UTOOL_CMD_QUERY_SCC_VERSION, ubctl_query_scc_data, ubctl_scc_version_deal}, + { UTOOL_CMD_QUERY_SCC_LOG, ubctl_query_scc_data, ubctl_scc_log_deal }, + + { UTOOL_CMD_QUERY_MSGQ_QUE_STATS, ubctl_query_msgq_que_stats_data, + ubctl_msgq_que_data_deal }, + { UTOOL_CMD_QUERY_MSGQ_ENTRY, ubctl_query_msgq_que_stats_data, + ubctl_msgq_entry_data_deal }, + + { UTOOL_CMD_QUERY_IO_DIE_PORT_INFO, ubctl_query_iodie_info_data, + ubctl_query_data_deal }, + + { UTOOL_CMD_QUERY_UMMU_ALL, ubctl_ummu_process_data, NULL }, + { UTOOL_CMD_QUERY_UMMU_SYNC, ubctl_ummu_process_data, NULL }, + { UTOOL_CMD_CONFIG_UMMU_SYNC, ubctl_ummu_process_data, NULL }, + + { UTOOL_CMD_QUERY_MAX, NULL, NULL } +}; + +struct ubctl_func_dispatch *ubctl_get_query_func(struct ubctl_dev *ucdev, u32 rpc_cmd) +{ + u32 i; + + if (!ucdev) + return NULL; + + for (i = 0; i < ARRAY_SIZE(g_ubctl_query_func); i++) { + if (g_ubctl_query_func[i].rpc_cmd == rpc_cmd) + return &g_ubctl_query_func[i]; + } + + return NULL; +} diff --git a/drivers/fwctl/ub/ub_cmd.h b/drivers/fwctl/ub/ub_cmd.h new file mode 100644 index 0000000000000000000000000000000000000000..69bb2ea43c52102e5abb13746b02ca8bbd2d91bd --- /dev/null +++ b/drivers/fwctl/ub/ub_cmd.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + */ + +#ifndef __UB_CMD_H__ +#define __UB_CMD_H__ + +#include "ub_common.h" + +struct ubctl_func_dispatch *ubctl_get_query_func(struct ubctl_dev *ucdev, + u32 rpc_cmd); +#endif diff --git a/drivers/fwctl/ub/ub_cmd_reg.c b/drivers/fwctl/ub/ub_cmd_reg.c new file mode 100644 index 0000000000000000000000000000000000000000..026ac3f2fe90ee706d8c445520bc07df6c056766 --- /dev/null +++ b/drivers/fwctl/ub/ub_cmd_reg.c @@ -0,0 +1,715 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + */ + +#include "ub_cmdq.h" +#include "ub_cmd_reg.h" + +static int ubctl_query_nl_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_NL_PKT_STATS_DFX, UBCTL_NL_PKT_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_NL_SSU_STATS_DFX, UBCTL_NL_SSU_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_NL_ABN_DFX, UBCTL_NL_ABN_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_nl_pkt_stats_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_NL_PKT_STATS_DFX, UBCTL_NL_PKT_STATS_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_nl_ssu_stats_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_NL_SSU_STATS_DFX, UBCTL_NL_SSU_STATS_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_nl_abn_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_NL_ABN_DFX, UBCTL_NL_ABN_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_dl_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_DL_PKT_STATS_DFX, UBCTL_DL_PKT_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_DL_REPL_DFX, UBCTL_DL_REPL_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_DL_LINK_STATUS_DFX, UBCTL_DL_LINK_STATUS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_DL_LANE_DFX, UBCTL_DL_LANE_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_DL_BIT_ERR_DFX, UBCTL_DL_BIT_ERR_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_dl_pkt_stats_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_DL_PKT_STATS_DFX, UBCTL_DL_PKT_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_DL_REPL_DFX, UBCTL_DL_REPL_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_dl_link_status_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_DL_LINK_STATUS_DFX, UBCTL_DL_LINK_STATUS_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_dl_lane_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_DL_LANE_DFX, UBCTL_DL_LANE_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_dl_bit_err_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_DL_BIT_ERR_DFX, UBCTL_DL_BIT_ERR_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_dl_bist_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_CONF_DL_BIST_DFX, UBCTL_DL_BIST_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_conf_dl_bist_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_CONF_DL_BIST_DFX, UBCTL_DL_BIST_LEN, UBCTL_WRITE, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_dl_bist_err_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_DL_BIST_ERR_DFX, UBCTL_DL_BIST_ERR_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_dp_deal(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func, + struct ubctl_query_dp *query_dp, u32 query_dp_num) +{ +#define UBCTL_TP_RX_BANK_NUM 3U + + u32 *rx_bank_id __free(kvfree) = kvzalloc(sizeof(u32) * UBCTL_TP_RX_BANK_NUM, GFP_KERNEL); + u32 bank_idx = 0; + u32 bank_id = 0; + int ret = 0; + u32 i; + + if (!rx_bank_id) + return -ENOMEM; + + for (i = 0; i < query_dp_num; i++) { + if (query_dp[i].op_code != UBCTL_QUERY_TP_RX_BANK_DFX) + continue; + if (bank_idx >= UBCTL_TP_RX_BANK_NUM) { + ubctl_err(ucdev, "bank_idx is out of bounds: %u.\n", bank_idx); + return -EINVAL; + } + + rx_bank_id[bank_idx] = bank_id++; + query_dp[i].data = (void *)&rx_bank_id[bank_idx++]; + query_dp[i].data_len = (u32)sizeof(u32); + } + + ret = ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, query_dp_num); + if (ret) + ubctl_err(ucdev, "ubctl query data failed, ret = %d.\n", ret); + + return ret; +} + +static int ubctl_query_tp_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_TP_TX_DFX, UBCTL_TP_TX_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RX_DFX, UBCTL_TP_RX_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RQM_DFX, UBCTL_TP_RQM_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_STATE_DFX, UBCTL_TP_STATE_DFX_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_TX_ROUTE_DFX, UBCTL_TP_TX_ROUTE_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RX_BANK_DFX, UBCTL_TP_RX_BANK_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RX_BANK_DFX, UBCTL_TP_RX_BANK_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RX_BANK_DFX, UBCTL_TP_RX_BANK_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_TX_DFX, UBCTL_TP_TX_ABN_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RX_DFX, UBCTL_TP_RX_ABN_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_ABN_STATS_DFX, UBCTL_TP_REG_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_dp_deal(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_tp_tx_route_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_TP_TX_ROUTE_DFX, UBCTL_TP_TX_ROUTE_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, query_dp, + ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_tp_abn_stats_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_TP_TX_DFX, UBCTL_TP_TX_ABN_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RX_DFX, UBCTL_TP_RX_ABN_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_ABN_STATS_DFX, UBCTL_TP_REG_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, query_dp, + ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_tp_pkt_stats_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_TP_TX_DFX, UBCTL_TP_TX_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RX_DFX, UBCTL_TP_RX_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RQM_DFX, UBCTL_TP_RQM_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_STATE_DFX, UBCTL_TP_STATE_DFX_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_tp_rx_bank_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_TP_RX_BANK_DFX, UBCTL_TP_RX_BANK_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RX_BANK_DFX, UBCTL_TP_RX_BANK_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RX_BANK_DFX, UBCTL_TP_RX_BANK_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_dp_deal(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_ta_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_TA_PKT_STATS_DFX, UBCTL_TA_PKT_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TA_ABN_STATS_DFX, UBCTL_TA_ABN_STATS_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_ta_pkt_stats(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_TA_PKT_STATS_DFX, UBCTL_TA_PKT_STATS_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_ta_abn_stats(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_TA_ABN_STATS_DFX, UBCTL_TA_ABN_STATS_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_ba_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_BA_PKT_STATS_DFX, UBCTL_BA_PKT_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_BA_MAR_DFX, UBCTL_BA_MAR_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_ba_pkt_stats_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_BA_PKT_STATS_DFX, UBCTL_BA_PKT_STATS_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_conf_ba_mar_perf(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_CONF_BA_PERF_DFX, UBCTL_CONF_BA_MAR_PERF_LEN, UBCTL_WRITE, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_ba_mar_perf(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_BA_MAR_PERF_DFX, UBCTL_QUERY_BA_MAR_PERF_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_ba_mar_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_BA_MAR_DFX, UBCTL_BA_MAR_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_mar_cyc_en_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_MAR_CYC_EN_DFX, UBCTL_MAR_CYC_EN_LEN, UBCTL_READ, NULL, 0 }, + }; + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_conf_mar_cyc_en_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_MAR_CYC_EN_DFX, UBCTL_MAR_CYC_EN_LEN, UBCTL_WRITE, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_mar_table_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ +#define UBCTL_UB_MEM_TABLE_ENTRY_LEN 16U +#define UBCTL_UB_MEM_TABLE_ENTRY_NUM 7U + + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_MAR_TABLE_DFX, UBCTL_MAR_TABLE_LEN, UBCTL_READ, NULL, 0 }, + }; + struct fwctl_pkt_in_table *mar_table = + (struct fwctl_pkt_in_table *)(query_cmd_param->in->data); + + if (query_cmd_param->in->data_size != sizeof(*mar_table)) { + ubctl_err(ucdev, "user data of mar table is invalid.\n"); + return -EINVAL; + } + + if (mar_table->table_num == UBCTL_UB_MEM_TABLE_ENTRY_NUM) + mar_table->index *= UBCTL_UB_MEM_TABLE_ENTRY_LEN; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_qos_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_QOS_DFX, UBCTL_QOS_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_scc_debug(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_SCC_DEBUG_DFX, UBCTL_SCC_DEBUG_EN_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_config_scc_debug(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_SCC_DEBUG_DFX, UBCTL_SCC_DEBUG_EN_LEN, UBCTL_WRITE, NULL, 0 }, + }; + + if (query_cmd_param->in->data_size != sizeof(struct fwctl_pkt_in_enable)) { + ubctl_err(ucdev, "user data of scc debug is invalid.\n"); + return -EINVAL; + } + u8 *scc_debug_en = (u8 *)(query_cmd_param->in->data); + + if (*scc_debug_en > 1) + return -EINVAL; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_ubommu_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_UBOMMU_DFX, UBCTL_UBOMMU_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_port_info_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_PORT_INFO_DFX, UBCTL_PORT_INFO_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, query_dp, + ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_ecc_2b_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_ECC_2B_DFX, UBCTL_ECC_2B_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, query_dp, + ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_queue_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_QUEUE_DFX, UBCTL_QUEUE_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, query_dp, + ARRAY_SIZE(query_dp)); +} + +static int ubctl_query_loopback(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_LOOPBACK, UBCTL_QUERY_DEBUG_EN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_config_loopback(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_LOOPBACK, UBCTL_QUERY_DEBUG_EN, UBCTL_WRITE, NULL, 0 }, + }; + int ret; + + ret = ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); + + if (query_cmd_param->out->retval == -EBUSY) + ubctl_err(ucdev, "Current port has been enabled for another loopback mode.\n"); + if (query_cmd_param->out->retval == -EMLINK) + ubctl_err(ucdev, "Another port has already been enabled.\n"); + + return ret; +} + +static int ubctl_query_prbs(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_PRBS_RESULT, UBCTL_QUERY_DEBUG_EN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static int ubctl_config_prbs(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_PRBS_RESULT, UBCTL_QUERY_DEBUG_EN, UBCTL_WRITE, NULL, 0 }, + }; + int ret; + + ret = ubctl_query_data(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); + + if (query_cmd_param->out->retval == -EMLINK) + ubctl_err(ucdev, "Another port has already been enabled.\n"); + + return ret; +} + +static int ubctl_query_dump_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func) +{ + struct ubctl_query_dp query_dp[] = { + { UBCTL_QUERY_NL_PKT_STATS_DFX, UBCTL_NL_PKT_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_NL_SSU_STATS_DFX, UBCTL_NL_SSU_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_NL_ABN_DFX, UBCTL_NL_ABN_LEN, UBCTL_READ, NULL, 0 }, + + { UBCTL_QUERY_TP_TX_DFX, UBCTL_TP_TX_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RX_DFX, UBCTL_TP_RX_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RQM_DFX, UBCTL_TP_RQM_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_STATE_DFX, UBCTL_TP_STATE_DFX_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_TX_ROUTE_DFX, UBCTL_TP_TX_ROUTE_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RX_BANK_DFX, UBCTL_TP_RX_BANK_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RX_BANK_DFX, UBCTL_TP_RX_BANK_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RX_BANK_DFX, UBCTL_TP_RX_BANK_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_TX_DFX, UBCTL_TP_TX_ABN_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_RX_DFX, UBCTL_TP_RX_ABN_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TP_ABN_STATS_DFX, UBCTL_TP_REG_LEN, UBCTL_READ, NULL, 0 }, + + { UBCTL_QUERY_TA_PKT_STATS_DFX, UBCTL_TA_PKT_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_TA_ABN_STATS_DFX, UBCTL_TA_ABN_STATS_LEN, UBCTL_READ, NULL, 0 }, + + { UBCTL_QUERY_DL_PKT_STATS_DFX, UBCTL_DL_PKT_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_DL_REPL_DFX, UBCTL_DL_REPL_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_DL_LINK_STATUS_DFX, UBCTL_DL_LINK_STATUS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_DL_LANE_DFX, UBCTL_DL_LANE_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_DL_BIT_ERR_DFX, UBCTL_DL_BIT_ERR_LEN, UBCTL_READ, NULL, 0 }, + + { UBCTL_QUERY_BA_PKT_STATS_DFX, UBCTL_BA_PKT_STATS_LEN, UBCTL_READ, NULL, 0 }, + { UBCTL_QUERY_BA_MAR_DFX, UBCTL_BA_MAR_LEN, UBCTL_READ, NULL, 0 }, + + { UBCTL_QUERY_QOS_DFX, UBCTL_QOS_LEN, UBCTL_READ, NULL, 0 }, + + { UBCTL_QUERY_UBOMMU_DFX, UBCTL_UBOMMU_LEN, UBCTL_READ, NULL, 0 }, + + { UBCTL_QUERY_ECC_2B_DFX, UBCTL_ECC_2B_LEN, UBCTL_READ, NULL, 0 }, + }; + + return ubctl_query_dp_deal(ucdev, query_cmd_param, query_func, + query_dp, ARRAY_SIZE(query_dp)); +} + +static struct ubctl_func_dispatch g_ubctl_query_reg[] = { + { UTOOL_CMD_QUERY_NL, ubctl_query_nl_data, ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_NL_PKT_STATS, ubctl_query_nl_pkt_stats_data, + ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_NL_SSU_STATS, ubctl_query_nl_ssu_stats_data, + ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_NL_ABN, ubctl_query_nl_abn_data, ubctl_query_data_deal }, + + { UTOOL_CMD_QUERY_DL, ubctl_query_dl_data, ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_DL_PKT_STATS, ubctl_query_dl_pkt_stats_data, + ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_DL_LINK_STATUS, ubctl_query_dl_link_status_data, + ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_DL_LANE, ubctl_query_dl_lane_data, + ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_DL_BIT_ERR, ubctl_query_dl_bit_err_data, + ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_DL_BIST, ubctl_query_dl_bist_data, + ubctl_query_data_deal }, + { UTOOL_CMD_CONF_DL_BIST, ubctl_conf_dl_bist_data, + ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_DL_BIST_ERR, ubctl_query_dl_bist_err_data, + ubctl_query_data_deal }, + + { UTOOL_CMD_QUERY_TP, ubctl_query_tp_data, ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_TP_PKT_STATS, ubctl_query_tp_pkt_stats_data, + ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_TP_ABN_STATS, ubctl_query_tp_abn_stats_data, + ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_TP_TX_ROUTE, ubctl_query_tp_tx_route_data, + ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_TP_RX_BANK, ubctl_query_tp_rx_bank_data, + ubctl_query_data_deal }, + + { UTOOL_CMD_QUERY_TA, ubctl_query_ta_data, ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_TA_PKT_STATS, ubctl_query_ta_pkt_stats, + ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_TA_ABN_STATS, ubctl_query_ta_abn_stats, + ubctl_query_data_deal }, + + { UTOOL_CMD_QUERY_BA, ubctl_query_ba_data, ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_BA_PKT_STATS, ubctl_query_ba_pkt_stats_data, + ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_BA_MAR, ubctl_query_ba_mar_data, ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_BA_MAR_TABLE, ubctl_query_mar_table_data, + ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_BA_MAR_CYC_EN, ubctl_query_mar_cyc_en_data, + ubctl_query_data_deal }, + { UTOOL_CMD_CONF_BA_MAR_CYC_EN, ubctl_conf_mar_cyc_en_data, + ubctl_query_data_deal }, + { UTOOL_CMD_CONFIG_BA_MAR_PEFR_STATS, ubctl_conf_ba_mar_perf, + ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_BA_MAR_PEFR_STATS, ubctl_query_ba_mar_perf, + ubctl_query_data_deal }, + + { UTOOL_CMD_QUERY_QOS, ubctl_query_qos_data, ubctl_query_data_deal }, + + { UTOOL_CMD_QUERY_SCC_DEBUG_EN, ubctl_query_scc_debug, + ubctl_query_data_deal }, + { UTOOL_CMD_CONF_SCC_DEBUG_EN, ubctl_config_scc_debug, + ubctl_query_data_deal }, + + { UTOOL_CMD_QUERY_UBOMMU, ubctl_query_ubommu_data, ubctl_query_data_deal }, + + { UTOOL_CMD_QUERY_PORT_INFO, ubctl_query_port_info_data, + ubctl_query_data_deal }, + + { UTOOL_CMD_QUERY_ECC_2B, ubctl_query_ecc_2b_data, ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_QUEUE, ubctl_query_queue_data, ubctl_query_data_deal }, + + { UTOOL_CMD_QUERY_LOOPBACK, ubctl_query_loopback, ubctl_query_data_deal }, + { UTOOL_CMD_CONF_LOOPBACK, ubctl_config_loopback, ubctl_query_data_deal }, + + { UTOOL_CMD_QUERY_PRBS_EN, ubctl_query_prbs, ubctl_query_data_deal }, + { UTOOL_CMD_CONF_PRBS_EN, ubctl_config_prbs, ubctl_query_data_deal }, + { UTOOL_CMD_QUERY_PRBS_RESULT, ubctl_query_prbs, ubctl_query_data_deal }, + + { UTOOL_CMD_QUERY_DUMP, ubctl_query_dump_data, ubctl_query_data_deal }, + + { UTOOL_CMD_QUERY_MAX, NULL, NULL } +}; + +struct ubctl_func_dispatch *ubctl_get_query_reg_func(struct ubctl_dev *ucdev, + u32 rpc_cmd) +{ + u32 i; + + if (!ucdev) + return NULL; + + for (i = 0; i < ARRAY_SIZE(g_ubctl_query_reg); i++) { + if (g_ubctl_query_reg[i].rpc_cmd == rpc_cmd) + return &g_ubctl_query_reg[i]; + } + + return NULL; +} diff --git a/drivers/fwctl/ub/ub_cmd_reg.h b/drivers/fwctl/ub/ub_cmd_reg.h new file mode 100644 index 0000000000000000000000000000000000000000..87b3e0183cd93bc7109831f7f1066f7434a63e25 --- /dev/null +++ b/drivers/fwctl/ub/ub_cmd_reg.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. 2025-2025. All rights reserved. + */ + +#ifndef __UB_CMD_REG_H__ +#define __UB_CMD_REG_H__ + +#include "ub_common.h" + +struct ubctl_func_dispatch *ubctl_get_query_reg_func(struct ubctl_dev *ucdev, + u32 rpc_cmd); +#endif diff --git a/drivers/fwctl/ub/ub_cmdq.h b/drivers/fwctl/ub/ub_cmdq.h new file mode 100644 index 0000000000000000000000000000000000000000..a8a4e63c42e3db3d2985f2f4efde14eb1d7be415 --- /dev/null +++ b/drivers/fwctl/ub/ub_cmdq.h @@ -0,0 +1,235 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + */ + +#ifndef __UB_CMDQ_H__ +#define __UB_CMDQ_H__ + +#define UBCTL_QUERY_NL_PKT_STATS_DFX 0xA001 +#define UBCTL_QUERY_NL_SSU_STATS_DFX 0xA002 +#define UBCTL_QUERY_NL_ABN_DFX 0xA003 + +#define UBCTL_QUERY_TP_TX_DFX 0xA004 +#define UBCTL_QUERY_TP_RX_DFX 0xA005 +#define UBCTL_QUERY_TP_TX_ROUTE_DFX 0xA01A +#define UBCTL_QUERY_TP_RX_BANK_DFX 0xA01C +#define UBCTL_QUERY_TP_ABN_STATS_DFX 0xA01D +#define UBCTL_QUERY_TP_RQM_DFX 0xA01E +#define UBCTL_QUERY_TP_STATE_DFX 0xA024 + +#define UBCTL_QUERY_TA_PKT_STATS_DFX 0xA006 +#define UBCTL_QUERY_TA_ABN_STATS_DFX 0xA023 + +#define UBCTL_QUERY_DL_PKT_STATS_DFX 0xA007 +#define UBCTL_QUERY_DL_LINK_STATUS_DFX 0xA008 +#define UBCTL_QUERY_DL_LANE_DFX 0xA009 +#define UBCTL_QUERY_DL_TRACE_DFX 0xA010 +#define UBCTL_QUERY_DL_BIT_ERR_DFX 0xA00A +#define UBCTL_QUERY_CONF_DL_BIST_DFX 0xA020 +#define UBCTL_QUERY_DL_BIST_ERR_DFX 0xA021 +#define UBCTL_QUERY_DL_REPL_DFX 0xA022 + +#define UBCTL_QUERY_BA_PKT_STATS_DFX 0xA00B +#define UBCTL_QUERY_BA_MAR_DFX 0xA00C +#define UBCTL_QUERY_MAR_TABLE_DFX 0xA012 +#define UBCTL_QUERY_MAR_CYC_EN_DFX 0xA013 +#define UBCTL_CONF_BA_PERF_DFX 0xA014 +#define UBCTL_QUERY_BA_MAR_PERF_DFX 0xA015 + +#define UBCTL_QUERY_QOS_DFX 0xA00D + +#define UBCTL_QUERY_SCC_DFX 0xA00E +#define UBCTL_QUERY_SCC_DEBUG_DFX 0xA011 + +#define UBCTL_QUERY_MSGQ_DFX 0xA00F +#define UBCTL_QUERY_QUEUE_DFX 0xA01B + +#define UBCTL_QUERY_UBOMMU_DFX 0xA016 + +#define UBCTL_QUERY_PORT_NUM_DFX 0xA017 +#define UBCTL_QUERY_PORT_INFO_DFX 0xA018 + +#define UBCTL_QUERY_ECC_2B_DFX 0xA019 + +#define UBCTL_QUERY_LOOPBACK 0xA025 +#define UBCTL_QUERY_PRBS_RESULT 0xA026 + +#define UBCTL_NL_PKT_STATS_LEN 632 +#define UBCTL_NL_SSU_STATS_LEN 408 +#define UBCTL_NL_ABN_LEN 56 + +#define UBCTL_TP_TX_STATS_LEN 904 +#define UBCTL_TP_RX_STATS_LEN 704 +#define UBCTL_TP_TX_ABN_LEN 948 +#define UBCTL_TP_RX_ABN_LEN 760 +#define UBCTL_TP_REG_LEN 24 +#define UBCTL_TP_TX_ROUTE_LEN 216 +#define UBCTL_TP_RX_BANK_LEN 408 +#define UBCTL_TP_RQM_LEN 88 +#define UBCTL_TP_STATE_DFX_LEN 376 + +#define UBCTL_TA_PKT_STATS_LEN 920 +#define UBCTL_TA_ABN_STATS_LEN 168 + +#define UBCTL_DL_PKT_STATS_LEN 984 +#define UBCTL_DL_REPL_LEN 120 +#define UBCTL_DL_LINK_STATUS_LEN 24 +#define UBCTL_DL_LANE_LEN 24 +#define UBCTL_DL_BIT_ERR_LEN 56 +#define UBCTL_DL_BIST_LEN 24 +#define UBCTL_DL_BIST_ERR_LEN 24 +#define UBCTL_DL_TRACE_LEN 1016 + +#define UBCTL_BA_PKT_STATS_LEN 792 +#define UBCTL_BA_MAR_LEN 440 +#define UBCTL_MAR_TABLE_LEN 88 +#define UBCTL_MAR_CYC_EN_LEN 24 +#define UBCTL_CONF_BA_MAR_PERF_LEN 24 +#define UBCTL_QUERY_BA_MAR_PERF_LEN 56 + +#define UBCTL_QOS_LEN 284 + +#define UBCTL_SCC_LEN 24 +#define UBCTL_SCC_DEBUG_EN_LEN 24 + +#define UBCTL_MSGQ_LEN 120 +#define UBCTL_QUEUE_LEN 120 + +#define UBCTL_IO_DIE_INFO_LEN 24 + +#define UBCTL_PORT_INFO_LEN 56 + +#define UBCTL_UBOMMU_LEN 56 + +#define UBCTL_ECC_2B_LEN 344 + +#define UBCTL_QUERY_DEBUG_EN 24 + +#define UBCTL_UMMU_CTRL0 0x0030 +#define UBCTL_UMMU_CTRL1 0x0038 +#define UBCTL_UMMU_TECT_BASE_CFG 0x0078 +#define UBCTL_UMMU_MCMD_QUE_PI_0 0x0108 +#define UBCTL_UMMU_MCMD_QUE_CI_0 0x010C +#define UBCTL_UMMU_MCMD_QUE_PI_1 0x0118 +#define UBCTL_UMMU_MCMD_QUE_CI_1 0x011C +#define UBCTL_UMMU_MCMD_QUE_PI_2 0x0128 +#define UBCTL_UMMU_MCMD_QUE_CI_2 0x012C +#define UBCTL_UMMU_MCMD_QUE_PI_3 0x0138 +#define UBCTL_UMMU_MCMD_QUE_CI_3 0x013C +#define UBCTL_UMMU_MCMD_QUE_PI_4 0x0148 +#define UBCTL_UMMU_MCMD_QUE_CI_4 0x014C +#define UBCTL_UMMU_MCMD_QUE_PI_5 0x0158 +#define UBCTL_UMMU_MCMD_QUE_CI_5 0x015C +#define UBCTL_UMMU_MCMD_QUE_PI_6 0x0168 +#define UBCTL_UMMU_MCMD_QUE_CI_6 0x016C +#define UBCTL_UMMU_MCMD_QUE_PI_7 0x0178 +#define UBCTL_UMMU_MCMD_QUE_CI_7 0x017C +#define UBCTL_UMMU_EVENT_QUE_PI 0x1108 +#define UBCTL_UMMU_EVENT_QUE_CI 0x110C +#define UBCTL_UMMU_EVENT_QUE_USI_ADDR0 0x1110 +#define UBCTL_UMMU_EVENT_QUE_USI_ADDR1 0x1114 +#define UBCTL_UMMU_GLB_INT_EN 0x1130 +#define UBCTL_UMMU_GLB_ERR_INT_USI_ADDR0 0x1140 +#define UBCTL_UMMU_GLB_ERR_INT_USI_ADDR1 0x1144 +#define UBCTL_UMMU_ERR_STATUS_0 0x2010 +#define UBCTL_UMMU_INT_MASK 0x3404 +#define UBCTL_UMMU_SYNC_TIMEOUT_OPEN 0x3410 +#define UBCTL_UMMU_SYNC_TIMEOUT_INFO 0x3418 +#define UBCTL_UMMU_SKY_QUEUE_STAT3_SP_0_63 0x4558 +#define UBCTL_UMMU_DFX_ECC_MONITOR_0 0x4D18 +#define UBCTL_UMMU_DFX_ECC_MONITOR_1 0x4D1C +#define UBCTL_UMMU_SPEC_DEF_DFX 0x4D60 +#define UBCTL_UMMU_DVM_RECEIVE_REQ_CNT 0x4D70 +#define UBCTL_UMMU_DVM_SEND_REQ_CNT 0x4D74 +#define UBCTL_UMMU_DVM_REQ_INFO0 0x4D78 +#define UBCTL_UMMU_DVM_REQ_INFO1 0x4D7C +#define UBCTL_UMMU_PMCG_INT_EN 0x5018 +#define UBCTL_UMMU_CFG_DFX_CFGBUS_STATUS 0x6000 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_SNP_ERR_CNT 0x6200 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_0 0x6204 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_1 0x6208 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_2 0x620C +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_3 0x6210 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_4 0x6214 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_5 0x6218 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_6 0x621C +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_7 0x6220 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_8 0x6224 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_9 0x6228 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_10 0x622C +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_11 0x6230 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_12 0x6234 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_13 0x6238 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_14 0x623C +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_ENTRY_STATUS_15 0x6240 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_SNP_STATUS 0x6280 +#define UBCTL_UMMU_SWIF_EVENTQ_DFX_DROP_CNT 0x6284 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_CTRL_STATUS1 0x6288 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_CMD_CTRL_STATUS2 0x628C +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_KCMD_STATUS 0x6290 +#define UBCTL_UMMU_SWIF_KCMDQ_DFX_KCMD_ERR_STATUS 0x6294 +#define UBCTL_UMMU_SWIF_UMCMD_DFX0 0x6300 +#define UBCTL_UMMU_SWIF_UMCMD_DFX1 0x6304 +#define UBCTL_UMMU_SWIF_UMCMD_DFX2 0x6308 +#define UBCTL_UMMU_SWIF_UMCMD_DFX3 0x630C +#define UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX0_0 0x6310 +#define UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX0_1 0x6314 +#define UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX0_2 0x6318 +#define UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX0_3 0x631C +#define UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX0_4 0x6320 +#define UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX0_5 0x6324 +#define UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX0_6 0x6328 +#define UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX1 0x632C +#define UBCTL_UMMU_SWIF_UMCMD_RR_WIN_DFX2 0x6330 +#define UBCTL_UMMU_SWIF_UMCMD_CACHE_DFX1 0x6334 +#define UBCTL_UMMU_SWIF_UMCMD_CACHE_DFX2 0x6338 +#define UBCTL_UMMU_SWIF_UMCMD_CACHE_DFX3 0x633C +#define UBCTL_UMMU_SWIF_UMCMD_CACHE_DFX4 0x6340 +#define UBCTL_UMMU_SWIF_UMCMD_CACHE_DFX5 0x6344 +#define UBCTL_UMMU_SWIF_UMCMD_CACHE_DFX6 0x6348 +#define UBCTL_UMMU_UBIF_DFX0 0x6400 +#define UBCTL_UMMU_UBIF_DFX1 0x6404 +#define UBCTL_UMMU_UBIF_DSTEID_DFX 0x640C +#define UBCTL_UMMU_UBIF_SYNC_DFX 0x6410 +#define UBCTL_UMMU_UBIF_KV_CACHE_NS_NSE_MISMATCH_DFX0 0x641C +#define UBCTL_UMMU_UBIF_KV_CACHE_NS_NSE_MISMATCH_DFX1 0x6420 +#define UBCTL_UMMU_UBIF_KV_CACHE_NS_NSE_MISMATCH_DFX2 0x6424 +#define UBCTL_UMMU_UBIF_KV_CACHE_NS_NSE_MISMATCH_DFX3 0x6428 +#define UBCTL_UMMU_UBIF_KV_CACHE_NS_NSE_MISMATCH_DFX4 0x642C +#define UBCTL_UMMU_TBU_TLB_LKUP_PROC 0x6600 +#define UBCTL_UMMU_TBU_TLB_STAT 0x6604 +#define UBCTL_UMMU_TBU_TLB_FAULT_CNT 0x6608 +#define UBCTL_UMMU_TBU_PLB_LKUP_PROC 0x660C +#define UBCTL_UMMU_TBU_PLB_STAT 0x6610 +#define UBCTL_UMMU_TBU_PLB_FAULT_CNT 0x6614 +#define UBCTL_UMMU_TBU_INVLD_MG_INFO 0x6618 +#define UBCTL_UMMU_TBU_RAB_STAT 0x661C +#define UBCTL_UMMU_TBU_RAB_ENTRY_INFO_0_7_15 0x6624 +#define UBCTL_UMMU_TBU_CNT 0x662C +#define UBCTL_UMMU_DFX_TBU_PERM_ERR_CNT 0x6634 +#define UBCTL_UMMU_TBU_DFX0 0x6638 +#define UBCTL_UMMU_TBU_DFX1 0x663C +#define UBCTL_UMMU_TCU_PTW_QUEUE_STAT_0_47 0x6804 +#define UBCTL_UMMU_TCU_PPTW_QUEUE_STAT_0_39 0x680C +#define UBCTL_UMMU_GPC_QUEUE_STAT_0_15 0x6814 +#define UBCTL_UMMU_ROOT_GPF_FAR_L 0x10028 +#define UBCTL_UMMU_ROOT_GPF_FAR_H 0x1002C + +#define UBCTL_UMMU_GPC_QUEUE_POINTER 0x6810 +#define UBCTL_UMMU_SKY_QUEUE_POINTER_SP 0x4540 +#define UBCTL_UMMU_TCU_PTW_QUEUE_POINTER 0x6800 +#define UBCTL_UMMU_TCU_PPTW_QUEUE_POINTER 0x6808 +#define UBCTL_UMMU_TBU_RAB_FUNC_EN 0x6620 + +#define UBCTL_UMMU_BANK_NUM 8 +#define UBCTL_UMMU_ENTRY_NUM 16 +#define UBCTL_UMMU_GPC_QUEUE_COUNT 16 +#define UBCTL_UMMU_TCU_PPTW_QUEUE_COUNT 40 +#define UBCTL_UMMU_TCU_PTW_QUEUE_COUNT 48 +#define UBCTL_UMMU_SKY_QUEUE_COUNT 64 + +#define UBCTL_UMMU_REGISTER_BASE 0 +#define UBCTL_UMMU_REGISTER_MAX_ADDR (UBCTL_UMMU_ROOT_GPF_FAR_H + 4U) + +#endif diff --git a/drivers/fwctl/ub/ub_common.c b/drivers/fwctl/ub/ub_common.c new file mode 100644 index 0000000000000000000000000000000000000000..23d67829c8de17197face0fbe6e2364993d91b5d --- /dev/null +++ b/drivers/fwctl/ub/ub_common.c @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. 2025-2025. All rights reserved. + */ + +#include + +#include "ub_common.h" +#include "ub_cmdq.h" + +static inline void ubctl_struct_cpu_to_le32(u32 *data, u32 cnt) +{ + for (u32 i = 0; i < cnt; i++) + data[i] = cpu_to_le32(data[i]); +} + +static inline void ubctl_struct_le32_to_cpu(u32 *data, u32 cnt) +{ + for (u32 i = 0; i < cnt; i++) + data[i] = le32_to_cpu(data[i]); +} + +static inline int ubctl_ubase_cmd_send_param_check(struct auxiliary_device *adev, + struct ubctl_cmd *cmd) +{ + if (!adev || !cmd) + return -EINVAL; + + if (!cmd->in_data || !cmd->out_data) + return -EINVAL; + + return 0; +} + +int ubctl_ubase_cmd_send(struct auxiliary_device *adev, struct ubctl_cmd *cmd) +{ + struct ubase_cmd_buf in, out; + int ret; + + if (ubctl_ubase_cmd_send_param_check(adev, cmd)) + return -EINVAL; + + ubctl_struct_cpu_to_le32(cmd->in_data, cmd->in_len / sizeof(u32)); + ubase_fill_inout_buf(&in, cmd->op_code, cmd->is_read, cmd->in_len, + cmd->in_data); + ubase_fill_inout_buf(&out, cmd->op_code, cmd->is_read, cmd->out_len, + cmd->out_data); + + ret = ubase_cmd_send_inout(adev, &in, &out); + if (ret) + return ret; + + ubctl_struct_le32_to_cpu(cmd->out_data, cmd->out_len / sizeof(u32)); + + return 0; +} + +int ubctl_fill_cmd(struct ubctl_cmd *cmd, void *cmd_in, void *cmd_out, + u32 out_len, u32 is_read) +{ + if (!cmd || !cmd_in || !cmd_out) + return -EINVAL; + + cmd->is_read = is_read; + cmd->in_data = cmd_in; + cmd->out_data = cmd_out; + cmd->in_len = out_len; + cmd->out_len = out_len; + + return 0; +} + +static int ubctl_query_param_check(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func, + struct ubctl_query_dp *query_dp) +{ + if (!ucdev || !query_cmd_param || !query_func || !query_dp) + return -EINVAL; + + if (!query_cmd_param->in || !query_cmd_param->out) { + ubctl_err(ucdev, "ubctl in or out is null.\n"); + return -EINVAL; + } + + if (!query_func->data_deal) { + ubctl_err(ucdev, "ubctl data deal func is null.\n"); + return -EINVAL; + } + + return 0; +} + +static int ubctl_cmd_send_deal(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_query_dp *query_dp, + struct ubctl_query_cmd_dp *cmd_data, u32 offset) +{ + int *retval = &query_cmd_param->out->retval; + struct ubctl_cmd cmd = {}; + int ret = 0; + + cmd.op_code = query_dp->op_code; + ret = ubctl_fill_cmd(&cmd, cmd_data->cmd_in, cmd_data->cmd_out, + query_dp->out_len, query_dp->is_read); + if (ret) { + ubctl_err(ucdev, "ubctl fill cmd failed.\n"); + return ret; + } + + *retval = ubctl_ubase_cmd_send(ucdev->adev, &cmd); + if (*retval) { + ubctl_err(ucdev, "ubctl ubase cmd send failed, retval = %d.\n", + *retval); + return -EINVAL; + } + + ret = cmd_data->query_func->data_deal(ucdev, query_cmd_param, &cmd, + query_dp->out_len, offset); + if (ret) + ubctl_err(ucdev, "ubctl data deal failed, ret = %d.\n", ret); + + return ret; +} + +static void ubctl_cmd_data_deal(struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_query_dp *query_dp, + struct ubctl_query_cmd_dp *cmd_dp) +{ + if (!query_dp->data) { + memcpy(cmd_dp->cmd_in, query_cmd_param->in->data, query_cmd_param->in->data_size); + return; + } + + if (query_dp->op_code == UBCTL_QUERY_TP_RX_BANK_DFX && + query_dp->data_len == (u32)sizeof(u32)) + memcpy(cmd_dp->cmd_in, query_dp->data, sizeof(u32)); +} + +int ubctl_query_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func, + struct ubctl_query_dp *query_dp, u32 query_dp_num) +{ + u32 offset = 0; + int ret = 0; + u32 i; + + ret = ubctl_query_param_check(ucdev, query_cmd_param, query_func, query_dp); + if (ret) { + ubctl_err(ucdev, "ubctl query param check failed, ret = %d.\n", ret); + return ret; + } + + for (i = 0; i < query_dp_num; i++) { + if (query_cmd_param->in->data_size > query_dp[i].out_len) { + ubctl_err(ucdev, "ubctl in data size is bigger than out len.\n"); + return -EINVAL; + } + + void *cmd_in __free(kvfree) = kvzalloc(query_dp[i].out_len, GFP_KERNEL); + if (!cmd_in) + return -ENOMEM; + + void *cmd_out __free(kvfree) = kvzalloc(query_dp[i].out_len, GFP_KERNEL); + if (!cmd_out) + return -ENOMEM; + + struct ubctl_query_cmd_dp cmd_dp = (struct ubctl_query_cmd_dp) { + .cmd_in = cmd_in, + .cmd_out = cmd_out, + .query_func = query_func, + }; + + ubctl_cmd_data_deal(query_cmd_param, &query_dp[i], &cmd_dp); + ret = ubctl_cmd_send_deal(ucdev, query_cmd_param, &query_dp[i], + &cmd_dp, offset); + if (ret) + return ret; + + offset += query_dp[i].out_len / sizeof(u32); + } + return 0; +} + +int ubctl_query_data_deal(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_cmd *cmd, u32 out_len, u32 offset) +{ + if (!ucdev || !query_cmd_param || !cmd) + return -EINVAL; + + if (!query_cmd_param->in || !query_cmd_param->out) { + ubctl_err(ucdev, "ubctl in or out is null.\n"); + return -EINVAL; + } + + if (cmd->out_len != out_len) { + ubctl_err(ucdev, "out data size is not equal to out len.\n"); + return -EINVAL; + } + + if ((offset * (u32)sizeof(u32) + out_len) > query_cmd_param->out_len) { + ubctl_err(ucdev, "offset size is bigger than user out len.\n"); + return -EINVAL; + } + + memcpy(&query_cmd_param->out->data[offset], cmd->out_data, cmd->out_len); + query_cmd_param->out->data_size += cmd->out_len; + + return 0; +} diff --git a/drivers/fwctl/ub/ub_common.h b/drivers/fwctl/ub/ub_common.h new file mode 100644 index 0000000000000000000000000000000000000000..ab10576e3914506d5f8a5da5018dee4eabce0f33 --- /dev/null +++ b/drivers/fwctl/ub/ub_common.h @@ -0,0 +1,145 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. + */ + +#ifndef __UB_COMMAND_H__ +#define __UB_COMMAND_H__ + +#include +#include +#include + +#include + +#define UBCTL_READ true +#define UBCTL_WRITE false + +#define ubctl_err(ucdev, format, ...) \ + dev_err(&ucdev->fwctl.dev, format, ##__VA_ARGS__) + +#define ubctl_dbg(ucdev, format, ...) \ + dev_dbg(&ucdev->fwctl.dev, "PID %u: " format, current->pid, \ + ##__VA_ARGS__) + +#define ubctl_info(ucdev, format, ...) \ + dev_info(&ucdev->fwctl.dev, "PID %u: " format, current->pid, \ + ##__VA_ARGS__) + +#define UBCTL_GET_PHY_ADDR(high, low) ((((u64)(high)) << 32) | (low)) +#define UBCTL_EXTRACT_BITS(value, start, end) \ + (((value) >> (start)) & ((1UL << ((end) - (start) + 1)) - 1)) + +/** + * struct ubctl_dev - Device struct of framework + * @fwctl: The device of fwctl + * @data_size: Length of @data + * @adev: data transmitted to users + */ +struct ubctl_dev { + struct fwctl_device fwctl; + DECLARE_KFIFO_PTR(ioctl_fifo, unsigned long); + struct auxiliary_device *adev; +}; + +/** + * struct ubctl_query_cmd_param - Parameters of userspace RPC + * @in_len: Length of @in + * @in: Data of input + * @out_len: Length of @out + * @out: Data of output + * + * Used to receive parameters passed from userspace RPC + */ +struct ubctl_query_cmd_param { + size_t in_len; + struct fwctl_rpc_ub_in *in; + size_t out_len; + struct fwctl_rpc_ub_out *out; +}; + +/** + * struct ubctl_cmd - Parameters of query command + * @op_code: The operation code + * @is_read: Read-only or read-write + * @in_len: Length of @in_data + * @out_len: Length of @out_data + * @in: Data of input + * @out: Data of output + * + * Used for sending and receiving software communication + */ +struct ubctl_cmd { + u32 op_code; + u32 is_read; + u32 in_len; + u32 out_len; + void *in_data; + void *out_data; +}; + +struct ubctl_func_dispatch { + u32 rpc_cmd; + int (*execute)(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func); + int (*data_deal)(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_cmd *cmd, u32 out_len, u32 offset_index); +}; + +struct ubctl_query_dp { + u32 op_code; + u32 out_len; + bool is_read; + void *data; + u32 data_len; +}; + +struct ubctl_query_cmd_dp { + struct ubctl_func_dispatch *query_func; + void *cmd_in; + void *cmd_out; +}; + +/** + * ubctl_ubase_cmd_send - The ubase interface for issuing cmdq + * @adev: The auxiliary framework device + * @cmd: Command information of ubctl + */ +int ubctl_ubase_cmd_send(struct auxiliary_device *adev, + struct ubctl_cmd *cmd); +int ubctl_fill_cmd(struct ubctl_cmd *cmd, void *cmd_in, void *cmd_out, + u32 out_len, u32 is_read); + +/** + * ubctl_query_data - Packaging and delivering parameters of cmdq + * @ucdev: Ubctl device + * @query_cmd_param: Parameters passed from userspace RPC + * @query_func: Callback functions for issuing and processing data + * @query_dp: Parameters related to cmdq + * @query_dp_num: Number of elements in @query_dp + * + */ +int ubctl_query_data(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_func_dispatch *query_func, + struct ubctl_query_dp *query_dp, u32 query_dp_num); + +/** + * ubctl_query_data_deal - Default callback function for processing returned data + * @ucdev: Ubctl device + * @query_cmd_param: Parameters passed from userspace RPC and IMP + * @cmd: Command information of ubctl + * @out_len: Data length of the 'out' in @query_cmd_param + * @offset: Data offset of the 'out' in @query_cmd_param + * + * On return the device is visible through sysfs and /dev, driver ops may be + * called. + */ +int ubctl_query_data_deal(struct ubctl_dev *ucdev, + struct ubctl_query_cmd_param *query_cmd_param, + struct ubctl_cmd *cmd, u32 out_len, u32 offset); + +#endif + diff --git a/drivers/net/ub/dev/ubl.c b/drivers/net/ub/dev/ubl.c index b3637476b1fbd154c47a238f1dbffcc1cab938fc..453e4a1c9ff52373a6c3aa54c38f5ac671c114f5 100644 --- a/drivers/net/ub/dev/ubl.c +++ b/drivers/net/ub/dev/ubl.c @@ -116,7 +116,7 @@ static const struct header_ops ubl_header_ops ____cacheline_aligned = { }; /** - * ubl_setup - setup ub link network device + * ubl_setup - set up a ubl netdev * @dev: network device * * Fill in the fields of the device structure with ubl-generic values. @@ -138,18 +138,17 @@ void ubl_setup(struct net_device *dev) EXPORT_SYMBOL(ubl_setup); /** - * alloc_ubldev_mqs - Allocates and sets up a ub-link device + * alloc_ubldev_mqs - allocate and set up a ubl netdev * @sizeof_priv: Size of additional driver-private structure to be allocated - * for this ubl device + * for this ubl netdev * @txqs: The number of TX queues this device has. * @rxqs: The number of RX queues this device has. * * Fill in the fields of the device structure with ubl-generic * values. Basically done everything except registering the device. * - * Constructs a new net device, completing with a private data area of - * size (sizeof_priv). A 32-byte (not bit) alignment is enforced for - * this private data area. + * Allocate a new net device, with a private data area of size (sizeof_priv). + * A 32-byte alignment is enforced for this private data area. */ struct net_device *alloc_ubldev_mqs(int sizeof_priv, unsigned int txqs, @@ -161,13 +160,13 @@ struct net_device *alloc_ubldev_mqs(int sizeof_priv, unsigned int txqs, EXPORT_SYMBOL(alloc_ubldev_mqs); /** - * ubl_type_trans - obtains skb->protocol and adds sw_ptype to the packet + * ubl_type_trans - obtain packet type and insert sw_ptype to the packet header * @skb: buffer to alter * @dev: source device * @type: packet type * - * Obtains the packet type and translates it to skb->protocol and adds sw_ptype - * to the packet data. + * Obtain the packet type, translate it to skb->protocol and insert sw_ptype + * to the packet header. */ __be16 ubl_type_trans(struct sk_buff *skb, struct net_device *dev, u8 type) { diff --git a/drivers/net/ub/unic/Makefile b/drivers/net/ub/unic/Makefile index 1419ee5695958226ceae194ceedf552dd3d7744f..041815219e0ca790a86610e637e350e71cdfc534 100644 --- a/drivers/net/ub/unic/Makefile +++ b/drivers/net/ub/unic/Makefile @@ -9,4 +9,5 @@ ccflags-y += -I$(srctree)/drivers/net/ub/unic/debugfs obj-$(CONFIG_UB_UNIC) += unic.o unic-objs = unic_main.o unic_ethtool.o unic_hw.o unic_guid.o unic_netdev.o unic_dev.o unic_qos_hw.o unic_event.o unic_crq.o unic-objs += unic_channel.o debugfs/unic_debugfs.o unic_rx.o unic_tx.o unic_txrx.o unic_comm_addr.o unic_rack_ip.o unic_stats.o +unic-objs += debugfs/unic_ctx_debugfs.o unic_reset.o debugfs/unic_qos_debugfs.o debugfs/unic_entry_debugfs.o unic-$(CONFIG_UB_UNIC_DCB) += unic_dcbnl.o diff --git a/drivers/net/ub/unic/debugfs/unic_ctx_debugfs.c b/drivers/net/ub/unic/debugfs/unic_ctx_debugfs.c new file mode 100644 index 0000000000000000000000000000000000000000..fc0d19fee0381f05cb906fec967e7857197eeecc --- /dev/null +++ b/drivers/net/ub/unic/debugfs/unic_ctx_debugfs.c @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + * + */ + +#include +#include +#include +#include + +#include "unic_ctx_debugfs.h" +#include "unic_debugfs.h" +#include "unic_dev.h" + +static inline void unic_jfs_ctx_titles_print(struct seq_file *s) +{ + seq_puts(s, "SQ_ID SQE_BB_SHIFT STATE JFS_MODE TX_JFCN\n"); +} + +static void unic_dump_jfs_ctx_info_sw(struct unic_sq *sq, struct seq_file *s, + u32 index) +{ + struct unic_jfs_ctx *ctx = &sq->jfs_ctx; + + seq_printf(s, "%-7u", index); + seq_printf(s, "%-14u", ctx->sqe_bb_shift); + seq_printf(s, "%-7u", ctx->state); + seq_printf(s, "%-10u", ctx->jfs_mode); + seq_printf(s, "%-9u\n", ctx->tx_jfcn); +} + +static inline void unic_jfr_ctx_titles_print(struct seq_file *s) +{ + seq_puts(s, "RQ_ID STATE RQE_SHIFT RX_JFCN PI CI"); + seq_puts(s, "RECORD_DB_EN\n"); +} + +static void unic_dump_jfr_ctx_info_sw(struct unic_rq *rq, struct seq_file *s, + u32 index) +{ + struct unic_jfr_ctx *ctx = &rq->jfr_ctx; + u32 jfcn; + + jfcn = ctx->jfcn_l | (ctx->jfcn_h << UNIC_JFR_JFCN_H_OFFSET); + + seq_printf(s, "%-7u", index); + seq_printf(s, "%-7u", ctx->state); + seq_printf(s, "%-11u", ctx->rqe_shift); + seq_printf(s, "%-9u", jfcn); + seq_printf(s, "%-7u", ctx->pi); + seq_printf(s, "%-7u", ctx->ci); + seq_printf(s, "%-14u\n", ctx->record_db_en); +} + +static inline void unic_jfc_ctx_titles_print(struct seq_file *s) +{ + seq_puts(s, "CQ_ID ARM_ST STATE INLINE_EN SHIFT CQE_COAL_CNT"); + seq_puts(s, "CEQN RECORD_DB_EN CQE_COAL_PEIRIOD\n"); +} + +static void unic_dump_jfc_ctx_info_sw(struct unic_cq *cq, struct seq_file *s, + u32 index) +{ + struct unic_jfc_ctx *ctx = &cq->jfc_ctx; + + seq_printf(s, "%-7u", index); + seq_printf(s, "%-8u", ctx->arm_st); + seq_printf(s, "%-7u", ctx->state); + seq_printf(s, "%-11u", ctx->inline_en); + seq_printf(s, "%-7u", ctx->shift); + seq_printf(s, "%-14u", ctx->cqe_coalesce_cnt); + seq_printf(s, "%-6u", ctx->ceqn); + seq_printf(s, "%-14u", ctx->record_db_en); + seq_printf(s, "%-18u\n", ctx->cqe_coalesce_period); +} + +static void unic_get_jfs_ctx_sw(struct unic_channels *channels, + struct seq_file *s, u32 index) +{ + struct unic_channel *channel = &channels->c[index]; + + unic_dump_jfs_ctx_info_sw(channel->sq, s, index); +} + +static void unic_get_jfr_ctx_sw(struct unic_channels *channels, + struct seq_file *s, u32 index) +{ + struct unic_channel *channel = &channels->c[index]; + + unic_dump_jfr_ctx_info_sw(channel->rq, s, index); +} + +static void unic_get_sq_jfc_ctx_sw(struct unic_channels *channels, + struct seq_file *s, u32 index) +{ + struct unic_channel *channel = &channels->c[index]; + + unic_dump_jfc_ctx_info_sw(channel->sq->cq, s, index); +} + +static void unic_get_rq_jfc_ctx_sw(struct unic_channels *channels, + struct seq_file *s, u32 index) +{ + struct unic_channel *channel = &channels->c[index]; + + unic_dump_jfc_ctx_info_sw(channel->rq->cq, s, index); +} + +enum unic_dbg_ctx_type { + UNIC_DBG_JFS_CTX = 0, + UNIC_DBG_JFR_CTX, + UNIC_DBG_SQ_JFC_CTX, + UNIC_DBG_RQ_JFC_CTX, +}; + +static int unic_dbg_dump_ctx_sw(struct seq_file *s, void *data, + enum unic_dbg_ctx_type ctx_type) +{ + struct unic_dbg_context { + void (*print_ctx_titles)(struct seq_file *s); + void (*get_ctx)(struct unic_channels *channels, struct seq_file *s, u32 index); + } dbg_ctx[] = { + {unic_jfs_ctx_titles_print, unic_get_jfs_ctx_sw}, + {unic_jfr_ctx_titles_print, unic_get_jfr_ctx_sw}, + {unic_jfc_ctx_titles_print, unic_get_sq_jfc_ctx_sw}, + {unic_jfc_ctx_titles_print, unic_get_rq_jfc_ctx_sw}, + }; + struct unic_dev *unic_dev = dev_get_drvdata(s->private); + int ret = 0; + u32 i; + + dbg_ctx[ctx_type].print_ctx_titles(s); + + if (!mutex_trylock(&unic_dev->channels.mutex)) + return -EBUSY; + + if (__unic_resetting(unic_dev) || !unic_dev->channels.c) { + ret = -EBUSY; + goto out; + } + + for (i = 0; i < unic_dev->channels.num; i++) + dbg_ctx[ctx_type].get_ctx(&unic_dev->channels, s, i); + +out: + mutex_unlock(&unic_dev->channels.mutex); + + return ret; +} + +int unic_dbg_dump_jfs_ctx_sw(struct seq_file *s, void *data) +{ + return unic_dbg_dump_ctx_sw(s, data, UNIC_DBG_JFS_CTX); +} + +int unic_dbg_dump_jfr_ctx_sw(struct seq_file *s, void *data) +{ + return unic_dbg_dump_ctx_sw(s, data, UNIC_DBG_JFR_CTX); +} + +int unic_dbg_dump_rq_jfc_ctx_sw(struct seq_file *s, void *data) +{ + return unic_dbg_dump_ctx_sw(s, data, UNIC_DBG_RQ_JFC_CTX); +} + +int unic_dbg_dump_sq_jfc_ctx_sw(struct seq_file *s, void *data) +{ + return unic_dbg_dump_ctx_sw(s, data, UNIC_DBG_SQ_JFC_CTX); +} + +struct unic_ctx_info { + u32 start_idx; + u32 ctx_size; + u8 op; + const char *ctx_name; +}; + +static int unic_get_ctx_info(struct unic_dev *unic_dev, + enum unic_dbg_ctx_type ctx_type, + struct unic_ctx_info *ctx_info) +{ + struct ubase_adev_caps *unic_caps = ubase_get_unic_caps(unic_dev->comdev.adev); + + if (!unic_caps) { + unic_err(unic_dev, "failed to get unic caps.\n"); + return -ENODATA; + } + + switch (ctx_type) { + case UNIC_DBG_JFS_CTX: + ctx_info->start_idx = unic_caps->jfs.start_idx; + ctx_info->ctx_size = UBASE_JFS_CTX_SIZE; + ctx_info->op = UBASE_MB_QUERY_JFS_CONTEXT; + ctx_info->ctx_name = "jfs"; + break; + case UNIC_DBG_JFR_CTX: + ctx_info->start_idx = unic_caps->jfr.start_idx; + ctx_info->ctx_size = UBASE_JFR_CTX_SIZE; + ctx_info->op = UBASE_MB_QUERY_JFR_CONTEXT; + ctx_info->ctx_name = "jfr"; + break; + case UNIC_DBG_SQ_JFC_CTX: + ctx_info->start_idx = unic_caps->jfc.start_idx; + ctx_info->ctx_size = UBASE_JFC_CTX_SIZE; + ctx_info->op = UBASE_MB_QUERY_JFC_CONTEXT; + ctx_info->ctx_name = "sq_jfc"; + break; + case UNIC_DBG_RQ_JFC_CTX: + ctx_info->start_idx = unic_caps->jfc.start_idx + + unic_dev->channels.num; + ctx_info->ctx_size = UBASE_JFC_CTX_SIZE; + ctx_info->op = UBASE_MB_QUERY_JFC_CONTEXT; + ctx_info->ctx_name = "rq_jfc"; + break; + default: + unic_err(unic_dev, "failed to get ctx info, ctx_type = %u.\n", + ctx_type); + return -ENODATA; + } + + return 0; +} + +static void unic_mask_jfs_ctx_key_words(void *buf) +{ + struct unic_jfs_ctx *jfs = (struct unic_jfs_ctx *)buf; + + jfs->sqe_token_id_l = 0; + jfs->sqe_token_id_h = 0; + jfs->sqe_base_addr_l = 0; + jfs->sqe_base_addr_h = 0; + jfs->sqe_pld_tokenid = 0; + jfs->rmt_tokenid = 0; + jfs->user_data_l = 0; + jfs->user_data_h = 0; +} + +static void unic_mask_jfr_ctx_key_words(void *buf) +{ + struct unic_jfr_ctx *jfr = (struct unic_jfr_ctx *)buf; + + jfr->rqe_token_id_l = 0; + jfr->rqe_token_id_h = 0; + jfr->rqe_base_addr_l = 0; + jfr->rqe_base_addr_h = 0; + jfr->pld_token_id = 0; + jfr->token_value = 0; + jfr->user_data_l = 0; + jfr->user_data_h = 0; + jfr->idx_que_addr_l = 0; + jfr->idx_que_addr_h = 0; + jfr->record_db_addr_l = 0; + jfr->record_db_addr_m = 0; + jfr->record_db_addr_h = 0; +} + +static void unic_mask_jfc_ctx_key_words(void *buf) +{ + struct unic_jfc_ctx *jfc = (struct unic_jfc_ctx *)buf; + + jfc->cqe_base_addr_l = 0; + jfc->cqe_base_addr_h = 0; + jfc->queue_token_id = 0; + jfc->record_db_addr_l = 0; + jfc->record_db_addr_h = 0; + jfc->rmt_token_id = 0; + jfc->remote_token_value = 0; +} + +static void unic_mask_ctx_key_words(void *buf, + enum unic_dbg_ctx_type ctx_type) +{ + switch (ctx_type) { + case UNIC_DBG_JFS_CTX: + unic_mask_jfs_ctx_key_words(buf); + break; + case UNIC_DBG_JFR_CTX: + unic_mask_jfr_ctx_key_words(buf); + break; + case UNIC_DBG_SQ_JFC_CTX: + case UNIC_DBG_RQ_JFC_CTX: + unic_mask_jfc_ctx_key_words(buf); + break; + default: + break; + } +} + +static int unic_dbg_dump_context_hw(struct seq_file *s, void *data, + enum unic_dbg_ctx_type ctx_type) +{ + struct unic_dev *unic_dev = dev_get_drvdata(s->private); + struct auxiliary_device *adev = unic_dev->comdev.adev; + struct unic_ctx_info ctx_info = {0}; + struct ubase_cmd_mailbox *mailbox; + struct ubase_mbx_attr attr = {0}; + int ret = 0; + u32 i; + + if (!mutex_trylock(&unic_dev->channels.mutex)) + return -EBUSY; + + if (__unic_resetting(unic_dev) || + !unic_dev->channels.c) { + ret = -EBUSY; + goto channel_ready_err; + } + + mailbox = ubase_alloc_cmd_mailbox(adev); + if (IS_ERR_OR_NULL(mailbox)) { + unic_err(unic_dev, "failed to alloc mailbox for dump context.\n"); + ret = -ENOMEM; + goto channel_ready_err; + } + + ret = unic_get_ctx_info(unic_dev, ctx_type, &ctx_info); + if (ret) + goto upgrade_ctx_err; + + for (i = 0; i < unic_dev->channels.num; i++) { + ubase_fill_mbx_attr(&attr, i + ctx_info.start_idx, ctx_info.op, + 0); + ret = ubase_hw_upgrade_ctx_ex(adev, &attr, mailbox); + if (ret) { + unic_err(unic_dev, + "failed to post query %s ctx mbx, ret = %d.\n", + ctx_info.ctx_name, ret); + goto upgrade_ctx_err; + } + + seq_printf(s, "offset\t%s", ctx_info.ctx_name); + seq_printf(s, "%u\n", i); + unic_mask_ctx_key_words(mailbox->buf, ctx_type); + ubase_print_context_hw(s, mailbox->buf, ctx_info.ctx_size); + seq_puts(s, "\n"); + } + +upgrade_ctx_err: + ubase_free_cmd_mailbox(adev, mailbox); +channel_ready_err: + mutex_unlock(&unic_dev->channels.mutex); + + return ret; +} + +int unic_dbg_dump_jfs_context_hw(struct seq_file *s, void *data) +{ + return unic_dbg_dump_context_hw(s, data, UNIC_DBG_JFS_CTX); +} + +int unic_dbg_dump_jfr_context_hw(struct seq_file *s, void *data) +{ + return unic_dbg_dump_context_hw(s, data, UNIC_DBG_JFR_CTX); +} + +int unic_dbg_dump_sq_jfc_context_hw(struct seq_file *s, void *data) +{ + return unic_dbg_dump_context_hw(s, data, UNIC_DBG_SQ_JFC_CTX); +} + +int unic_dbg_dump_rq_jfc_context_hw(struct seq_file *s, void *data) +{ + return unic_dbg_dump_context_hw(s, data, UNIC_DBG_RQ_JFC_CTX); +} diff --git a/drivers/net/ub/unic/debugfs/unic_ctx_debugfs.h b/drivers/net/ub/unic/debugfs/unic_ctx_debugfs.h new file mode 100644 index 0000000000000000000000000000000000000000..cb6b08c80d8e96b660a64fdba44c7678335721df --- /dev/null +++ b/drivers/net/ub/unic/debugfs/unic_ctx_debugfs.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + * + */ + +#ifndef __UNIC_CTX_DEBUGFS_H__ +#define __UNIC_CTX_DEBUGFS_H__ + +int unic_dbg_dump_jfs_ctx_sw(struct seq_file *s, void *data); +int unic_dbg_dump_jfr_ctx_sw(struct seq_file *s, void *data); +int unic_dbg_dump_sq_jfc_ctx_sw(struct seq_file *s, void *data); +int unic_dbg_dump_rq_jfc_ctx_sw(struct seq_file *s, void *data); +int unic_dbg_dump_jfs_context_hw(struct seq_file *s, void *data); +int unic_dbg_dump_jfr_context_hw(struct seq_file *s, void *data); +int unic_dbg_dump_sq_jfc_context_hw(struct seq_file *s, void *data); +int unic_dbg_dump_rq_jfc_context_hw(struct seq_file *s, void *data); + +#endif diff --git a/drivers/net/ub/unic/debugfs/unic_debugfs.c b/drivers/net/ub/unic/debugfs/unic_debugfs.c index 112e64c25a3487002dde49aa5454315f20bd223a..4c7bf3bb83fccde9b25efbeefad09565f8314ae0 100644 --- a/drivers/net/ub/unic/debugfs/unic_debugfs.c +++ b/drivers/net/ub/unic/debugfs/unic_debugfs.c @@ -10,8 +10,11 @@ #include #include +#include "unic_ctx_debugfs.h" #include "unic_dev.h" #include "unic_hw.h" +#include "unic_qos_debugfs.h" +#include "unic_entry_debugfs.h" #include "unic_debugfs.h" static int unic_dbg_dump_dev_info(struct seq_file *s, void *data) @@ -36,6 +39,61 @@ static int unic_dbg_dump_dev_info(struct seq_file *s, void *data) return 0; } +static int unic_dbg_dump_vport_buf(struct seq_file *s, void *data) +{ + struct unic_dev *unic_dev = dev_get_drvdata(s->private); + + seq_printf(s, "vport buffer num: %u\n", unic_dev->caps.vport_buf_num); + seq_printf(s, "vport buffer size: %u\n", unic_dev->caps.vport_buf_size); + return 0; +} + +static void unic_dbg_fill_vport_ctx_content(struct unic_vport_ctx_cmd *resp, + struct seq_file *s) +{ + u32 i, j; + + for (i = 0; i < UNIC_VORT_CTX_DATA_NUM; i += UNIC_VORT_CTX_DATA_ALIGN) { + seq_printf(s, "%08X: ", i * UNIC_VORT_CTX_DATA_ALIGN); + for (j = 0; j < UNIC_VORT_CTX_DATA_ALIGN; j++) { + if ((i + j) == UNIC_VORT_CTX_DATA_NUM) + break; + seq_printf(s, "%08X ", resp->data[i + j]); + } + seq_puts(s, "\n"); + } +} + +static int unic_dbg_query_vport_ctx(struct seq_file *s) +{ + struct unic_dev *unic_dev = dev_get_drvdata(s->private); + struct unic_vport_ctx_cmd resp; + u16 offset = 0; + int ret; + + do { + memset(&resp, 0, sizeof(resp)); + ret = unic_query_vport_ctx(unic_dev, offset, &resp); + if (ret) + return ret; + offset = resp.offset; + + unic_dbg_fill_vport_ctx_content(&resp, s); + } while (resp.offset); + + return 0; +} + +static int unic_dbg_dump_vport_ctx(struct seq_file *s, void *data) +{ + struct unic_dev *unic_dev = dev_get_drvdata(s->private); + + if (__unic_resetting(unic_dev)) + return -EBUSY; + + return unic_dbg_query_vport_ctx(s); +} + static const struct unic_dbg_cap_bit_info { const char *format; bool (*get_bit)(struct unic_dev *dev); @@ -43,7 +101,6 @@ static const struct unic_dbg_cap_bit_info { {"\tsupport_ubl: %u\n", &unic_dev_ubl_supported}, {"\tsupport_ets: %u\n", &unic_dev_ets_supported}, {"\tsupport_fec: %u\n", &unic_dev_fec_supported}, - {"\tsupport_rss: %u\n", &unic_dev_rss_supported}, {"\tsupport_tc_speed_limit: %u\n", &unic_dev_tc_speed_limit_supported}, {"\tsupport_tx_csum_offload: %u\n", &unic_dev_tx_csum_offload_supported}, {"\tsupport_rx_csum_offload: %u\n", &unic_dev_rx_csum_offload_supported}, @@ -190,6 +247,63 @@ static int unic_dbg_dump_promisc_cfg_hw(struct seq_file *s, void *data) return 0; } +static int unic_dbg_query_link_record(struct seq_file *s, void *data) +{ + struct unic_dev *unic_dev = dev_get_drvdata(s->private); + struct unic_link_stats *record = &unic_dev->stats.link_record; + u8 cnt = 1, stats_cnt; + u64 total, idx; + + mutex_lock(&record->lock); + + seq_puts(s, "current time : "); + ubase_dbg_format_time(ktime_get_real_seconds(), s); + seq_printf(s, "\nlink up count : %llu\n", record->link_up_cnt); + seq_printf(s, "link down count : %llu\n", record->link_down_cnt); + + total = record->link_up_cnt + record->link_down_cnt; + if (!total) { + seq_puts(s, "link change records : NA\n"); + mutex_unlock(&record->lock); + + return 0; + } + + seq_puts(s, "link change records :\n"); + seq_puts(s, "\tNo.\tTIME\t\t\t\tSTATUS\n"); + + stats_cnt = min(total, LINK_STAT_MAX_IDX); + while (cnt <= stats_cnt) { + total--; + idx = total % LINK_STAT_MAX_IDX; + seq_printf(s, "\t%-2d\t", cnt); + ubase_dbg_format_time(record->stats[idx].link_tv_sec, s); + seq_printf(s, "\t%s\n", + record->stats[idx].link_status ? "LINK UP" : "LINK DOWN"); + cnt++; + } + + mutex_unlock(&record->lock); + + return 0; +} + +static int unic_dbg_clear_link_record(struct seq_file *s, void *data) +{ + struct unic_dev *unic_dev = dev_get_drvdata(s->private); + struct unic_link_stats *record = &unic_dev->stats.link_record; + + mutex_lock(&record->lock); + record->link_up_cnt = 0; + record->link_down_cnt = 0; + memset(record->stats, 0, sizeof(record->stats)); + mutex_unlock(&record->lock); + + seq_puts(s, "Link status records have been cleared!\n"); + + return 0; +} + static bool unic_dbg_dentry_support(struct device *dev, u32 property) { struct unic_dev *unic_dev = dev_get_drvdata(dev); @@ -198,6 +312,23 @@ static bool unic_dbg_dentry_support(struct device *dev, u32 property) } static struct ubase_dbg_dentry_info unic_dbg_dentry[] = { + { + .name = "ip_tbl", + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + }, { + .name = "context", + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + }, { + .name = "vport", + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + }, { + .name = "qos", + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + }, /* keep unic at the bottom and add new directory above */ { .name = "unic", @@ -208,12 +339,68 @@ static struct ubase_dbg_dentry_info unic_dbg_dentry[] = { static struct ubase_dbg_cmd_info unic_dbg_cmd[] = { { + .name = "ip_tbl_spec", + .dentry_index = UNIC_DBG_DENTRY_IP, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_ip_tbl_spec, + }, { + .name = "ip_tbl_list", + .dentry_index = UNIC_DBG_DENTRY_IP, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_ip_tbl_list, + }, { + .name = "jfs_context", + .dentry_index = UNIC_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_jfs_ctx_sw, + }, { + .name = "jfr_context", + .dentry_index = UNIC_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_jfr_ctx_sw, + }, { + .name = "sq_jfc_context", + .dentry_index = UNIC_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_sq_jfc_ctx_sw, + }, { + .name = "rq_jfc_context", + .dentry_index = UNIC_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_rq_jfc_ctx_sw, + }, { .name = "dev_info", .dentry_index = UNIC_DBG_DENTRY_ROOT, .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, .support = unic_dbg_dentry_support, .init = ubase_dbg_seq_file_init, .read_func = unic_dbg_dump_dev_info, + }, { + .name = "vport_buf", + .dentry_index = UNIC_DBG_DENTRY_VPORT, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL_ETH, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_vport_buf, + }, { + .name = "vport_ctx", + .dentry_index = UNIC_DBG_DENTRY_VPORT, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL_ETH, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_vport_ctx, }, { .name = "caps_info", .dentry_index = UNIC_DBG_DENTRY_ROOT, @@ -228,6 +415,41 @@ static struct ubase_dbg_cmd_info unic_dbg_cmd[] = { .support = unic_dbg_dentry_support, .init = ubase_dbg_seq_file_init, .read_func = unic_dbg_dump_page_pool_info, + }, { + .name = "jfs_context_hw", + .dentry_index = UNIC_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_jfs_context_hw, + }, { + .name = "jfr_context_hw", + .dentry_index = UNIC_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_jfr_context_hw, + }, { + .name = "sq_jfc_context_hw", + .dentry_index = UNIC_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_sq_jfc_context_hw, + }, { + .name = "rq_jfc_context_hw", + .dentry_index = UNIC_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_rq_jfc_context_hw, + }, { + .name = "vl_queue", + .dentry_index = UNIC_DBG_DENTRY_QOS, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_vl_queue, }, { .name = "rss_cfg_hw", .dentry_index = UNIC_DBG_DENTRY_ROOT, @@ -242,6 +464,41 @@ static struct ubase_dbg_cmd_info unic_dbg_cmd[] = { .support = unic_dbg_dentry_support, .init = ubase_dbg_seq_file_init, .read_func = unic_dbg_dump_promisc_cfg_hw, + }, { + .name = "dscp_vl_map", + .dentry_index = UNIC_DBG_DENTRY_QOS, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_dscp_vl_map, + }, { + .name = "prio_vl_map", + .dentry_index = UNIC_DBG_DENTRY_QOS, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_prio_vl_map, + }, { + .name = "dscp_prio", + .dentry_index = UNIC_DBG_DENTRY_QOS, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_dump_dscp_prio, + }, { + .name = "link_status_record", + .dentry_index = UNIC_DBG_DENTRY_ROOT, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_query_link_record, + }, { + .name = "clear_link_status_record", + .dentry_index = UNIC_DBG_DENTRY_ROOT, + .property = UBASE_SUP_UNIC | UBASE_SUP_UBL, + .support = unic_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = unic_dbg_clear_link_record, } }; diff --git a/drivers/net/ub/unic/debugfs/unic_debugfs.h b/drivers/net/ub/unic/debugfs/unic_debugfs.h index 6efd739d2c5b303e3ba40d27687abe8390b833b0..853597b90f45911a263d00d47e25107ae10740de 100644 --- a/drivers/net/ub/unic/debugfs/unic_debugfs.h +++ b/drivers/net/ub/unic/debugfs/unic_debugfs.h @@ -13,6 +13,10 @@ #define unic_get_ubase_root_dentry(adev) ubase_diag_debugfs_root(adev) enum unic_dbg_dentry_type { + UNIC_DBG_DENTRY_IP = 0, + UNIC_DBG_DENTRY_CONTEXT, + UNIC_DBG_DENTRY_VPORT, + UNIC_DBG_DENTRY_QOS, /* must be the last entry. */ UNIC_DBG_DENTRY_ROOT }; diff --git a/drivers/net/ub/unic/debugfs/unic_entry_debugfs.c b/drivers/net/ub/unic/debugfs/unic_entry_debugfs.c new file mode 100644 index 0000000000000000000000000000000000000000..74b5fdb95aaa311966557beb15122daf471224b3 --- /dev/null +++ b/drivers/net/ub/unic/debugfs/unic_entry_debugfs.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + * + */ + +#include +#include + +#include "unic_comm_addr.h" +#include "unic_dev.h" +#include "unic_debugfs.h" +#include "unic_entry_debugfs.h" + +static const char * const unic_entry_state_str[] = { + "TO_ADD", "TO_DEL", "ACTIVE" +}; + +int unic_dbg_dump_ip_tbl_spec(struct seq_file *s, void *data) +{ + struct unic_dev *unic_dev = dev_get_drvdata(s->private); + u32 total_ip_tbl_size, total_ue_num; + struct ubase_caps *ubase_caps; + + ubase_caps = ubase_get_dev_caps(unic_dev->comdev.adev); + total_ue_num = ubase_caps->total_ue_num; + total_ip_tbl_size = unic_dev->caps.total_ip_tbl_size; + + seq_printf(s, "total_ue_num\t: %u\n", total_ue_num); + seq_printf(s, "total_ip_tbl_size\t: %u\n", total_ip_tbl_size); + + return 0; +} + +int unic_dbg_dump_ip_tbl_list(struct seq_file *s, void *data) +{ + struct unic_dev *unic_dev = dev_get_drvdata(s->private); + struct unic_comm_addr_node *ip_node; + struct unic_addr_tbl *ip_tbl; + struct list_head *list; + u16 i = 0; + + seq_printf(s, "No %-43sSTATE IP_MASK\n", "IP_ADDR"); + + ip_tbl = &unic_dev->vport.addr_tbl; + list = &ip_tbl->ip_list; + spin_lock_bh(&ip_tbl->ip_list_lock); + list_for_each_entry(ip_node, list, node) { + seq_printf(s, "%-4d", i++); + seq_printf(s, "%-43pI6c", &ip_node->ip_addr.s6_addr); + seq_printf(s, "%-9s", unic_entry_state_str[ip_node->state]); + seq_printf(s, "%-3u", ip_node->node_mask); + + seq_puts(s, "\n"); + } + spin_unlock_bh(&ip_tbl->ip_list_lock); + + return 0; +} diff --git a/drivers/net/ub/unic/debugfs/unic_entry_debugfs.h b/drivers/net/ub/unic/debugfs/unic_entry_debugfs.h new file mode 100644 index 0000000000000000000000000000000000000000..73ab85f4d5f36b8e2a3fc17a065e8b8dcfdf99d4 --- /dev/null +++ b/drivers/net/ub/unic/debugfs/unic_entry_debugfs.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + * + */ + +#ifndef __UNIC_ENTRY_DEBUGFS_H__ +#define __UNIC_ENTRY_DEBUGFS_H__ + +#include +#include + +int unic_dbg_dump_ip_tbl_spec(struct seq_file *s, void *data); +int unic_dbg_dump_ip_tbl_list(struct seq_file *s, void *data); + +#endif /* _UNIC_ENTRY_DEBUGFS_H */ diff --git a/drivers/net/ub/unic/debugfs/unic_qos_debugfs.c b/drivers/net/ub/unic/debugfs/unic_qos_debugfs.c new file mode 100644 index 0000000000000000000000000000000000000000..85b3288f0becee866d30fd6559d5def6595438f4 --- /dev/null +++ b/drivers/net/ub/unic/debugfs/unic_qos_debugfs.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + * + */ + +#include "unic_dcbnl.h" +#include "unic_debugfs.h" +#include "unic_hw.h" +#include "unic_qos_debugfs.h" + +int unic_dbg_dump_vl_queue(struct seq_file *s, void *data) +{ + struct unic_dev *unic_dev = dev_get_drvdata(s->private); + struct unic_vl *vl = &unic_dev->channels.vl; + u8 i; + + seq_puts(s, "VL_ID Q_OFFSET Q_COUNT\n"); + + for (i = 0; i < unic_dev->channels.rss_vl_num; i++) { + seq_printf(s, "%-7d", i); + seq_printf(s, "%-12u", vl->queue_offset[i]); + seq_printf(s, "%-11u\n", vl->queue_count[i]); + } + + return 0; +} + +static void unic_dump_dscp_vl_map(struct unic_dev *unic_dev, + struct seq_file *s, u8 dscp, u8 hw_vl) +{ + struct auxiliary_device *adev = unic_dev->comdev.adev; + struct ubase_adev_qos *qos = ubase_get_adev_qos(adev); + struct unic_vl *vl = &unic_dev->channels.vl; + u8 prio, sw_vl = 0; + + prio = vl->dscp_prio[dscp]; + if (prio == UNIC_INVALID_PRIORITY || + vl->prio_vl[prio] >= unic_dev->channels.rss_vl_num) + sw_vl = qos->nic_vl[0]; + else + sw_vl = qos->nic_vl[vl->prio_vl[prio]]; + + seq_printf(s, "%-6u", dscp); + seq_printf(s, "%-7u", sw_vl); + + if (!unic_dev_ubl_supported(unic_dev) && + unic_dev_ets_supported(unic_dev)) + seq_printf(s, "%-7u", hw_vl); + else + seq_puts(s, "--"); + + seq_puts(s, "\n"); +} + +int unic_dbg_dump_dscp_vl_map(struct seq_file *s, void *data) +{ + struct unic_dev *unic_dev = dev_get_drvdata(s->private); + struct unic_config_vl_map_cmd resp = {0}; + int ret; + u8 i; + + if (__unic_resetting(unic_dev)) + return -EBUSY; + + if (!unic_dev_ubl_supported(unic_dev) && + unic_dev_ets_supported(unic_dev)) { + ret = unic_query_vl_map(unic_dev, &resp); + if (ret) + return ret; + } + + seq_puts(s, "DSCP SW_VL HW_VL\n"); + + for (i = 0; i < UBASE_MAX_DSCP; i++) + unic_dump_dscp_vl_map(unic_dev, s, i, resp.dscp_vl[i]); + + return 0; +} + +static void unic_dump_prio_vl_map(struct unic_dev *unic_dev, + struct seq_file *s, u8 prio, u8 hw_vl) +{ + struct auxiliary_device *adev = unic_dev->comdev.adev; + struct ubase_adev_qos *qos = ubase_get_adev_qos(adev); + struct unic_vl *vl = &unic_dev->channels.vl; + u8 sw_vl = 0; + + if (vl->prio_vl[prio] >= unic_dev->channels.rss_vl_num) + sw_vl = qos->nic_vl[0]; + else + sw_vl = qos->nic_vl[vl->prio_vl[prio]]; + + seq_printf(s, "%-6u", prio); + seq_printf(s, "%-7u", sw_vl); + + if (!unic_dev_ubl_supported(unic_dev) && + unic_dev_ets_supported(unic_dev)) + seq_printf(s, "%-7u", hw_vl); + else + seq_puts(s, "--"); + + seq_puts(s, "\n"); +} + +int unic_dbg_dump_prio_vl_map(struct seq_file *s, void *data) +{ + struct unic_dev *unic_dev = dev_get_drvdata(s->private); + struct unic_config_vl_map_cmd resp = {0}; + int ret; + u8 i; + + if (__unic_resetting(unic_dev)) + return -EBUSY; + + if (!unic_dev_ubl_supported(unic_dev) && + unic_dev_ets_supported(unic_dev)) { + ret = unic_query_vl_map(unic_dev, &resp); + if (ret) + return ret; + } + + seq_puts(s, "PRIO SW_VL HW_VL\n"); + + for (i = 0; i < UNIC_MAX_PRIO_NUM; i++) + unic_dump_prio_vl_map(unic_dev, s, i, resp.prio_vl[i]); + + return 0; +} + +int unic_dbg_dump_dscp_prio(struct seq_file *s, void *data) +{ + struct unic_dev *unic_dev = dev_get_drvdata(s->private); + struct unic_vl *vl = &unic_dev->channels.vl; + u16 i; + + if (__unic_resetting(unic_dev)) + return -EBUSY; + + seq_puts(s, "DSCP PRIO\n"); + + for (i = 0; i < UBASE_MAX_DSCP; i++) { + seq_printf(s, "%-6u", i); + seq_printf(s, "%-7u\n", vl->dscp_prio[i]); + } + + return 0; +} diff --git a/drivers/net/ub/unic/debugfs/unic_qos_debugfs.h b/drivers/net/ub/unic/debugfs/unic_qos_debugfs.h new file mode 100644 index 0000000000000000000000000000000000000000..f55616ab16174a9428a1e7f60441602a51751b52 --- /dev/null +++ b/drivers/net/ub/unic/debugfs/unic_qos_debugfs.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + * + */ + +#ifndef __UNIC_QOS_DEBUGFS_H__ +#define __UNIC_QOS_DEBUGFS_H__ + +int unic_dbg_dump_vl_queue(struct seq_file *s, void *data); +int unic_dbg_dump_dscp_vl_map(struct seq_file *s, void *data); +int unic_dbg_dump_prio_vl_map(struct seq_file *s, void *data); +int unic_dbg_dump_dscp_prio(struct seq_file *s, void *data); + +#endif diff --git a/drivers/net/ub/unic/unic.h b/drivers/net/ub/unic/unic.h index 7f6572c50a0c859fa78867d5b8724bbbc9b8af32..e63ee6e900ffe436f6f4e9d7fbb0e3770be16ec4 100644 --- a/drivers/net/ub/unic/unic.h +++ b/drivers/net/ub/unic/unic.h @@ -20,16 +20,11 @@ enum { UNIC_SUPPORT_ETS_B = 1, UNIC_SUPPORT_FEC_B = 2, UNIC_SUPPORT_PAUSE_B = 3, - UNIC_SUPPORT_GRO_B = 5, UNIC_SUPPORT_ETH_B = 7, - UNIC_SUPPORT_TSO_B = 8, - UNIC_SUPPORT_RSS_B = 9, UNIC_SUPPORT_SERIAL_SERDES_LB_B = 10, UNIC_SUPPORT_TC_SPEED_LIMIT_B = 12, UNIC_SUPPORT_TX_CSUM_OFFLOAD_B = 13, - UNIC_SUPPORT_TUNNEL_CSUM_OFFLOAD_B = 14, - UNIC_SUPPORT_PTP_B = 15, UNIC_SUPPORT_RX_CSUM_OFFLOAD_B = 16, UNIC_SUPPORT_APP_LB_B = 17, diff --git a/drivers/net/ub/unic/unic_cmd.h b/drivers/net/ub/unic/unic_cmd.h index bf3e11e343cd140a4192d8908438e23431b40914..ac571815be6aef863e9db67076a6f4a44e391e91 100644 --- a/drivers/net/ub/unic/unic_cmd.h +++ b/drivers/net/ub/unic/unic_cmd.h @@ -109,6 +109,14 @@ struct unic_cfg_vport_buf_cmd { __le32 buf_addr[UNIC_MAX_VPORT_BUF_NUM * U32S_PER_U64]; }; +#define UNIC_VORT_CTX_DATA_NUM 13 +#define UNIC_VORT_CTX_DATA_ALIGN 4 +struct unic_vport_ctx_cmd { + u8 resv[2]; + __le16 offset; + __le32 data[UNIC_VORT_CTX_DATA_NUM]; +}; + struct unic_cfg_fec_cmd { __le32 fec_mode; u8 rsv[20]; @@ -150,7 +158,7 @@ struct unic_config_vl_map_cmd { }; struct unic_config_vl_speed_cmd { - __le16 bus_ue_id; + u8 resv0[2]; __le16 vl_bitmap; __le32 max_speed[UBASE_MAX_VL_NUM]; u8 resv1[20]; diff --git a/drivers/net/ub/unic/unic_dcbnl.c b/drivers/net/ub/unic/unic_dcbnl.c index 3110277da020345f7fb62bfbccef8010a27dea82..5f14cebac540e64251c4a094e9b113dedd3f6c90 100644 --- a/drivers/net/ub/unic/unic_dcbnl.c +++ b/drivers/net/ub/unic/unic_dcbnl.c @@ -5,12 +5,487 @@ */ #include +#include #include "unic_hw.h" #include "unic_netdev.h" #include "unic_dcbnl.h" +#define UNIC_PRIO_VL_MAP_CHANGED BIT(0) +#define UNIC_TC_TSA_CHANGED BIT(1) +#define UNIC_TC_BW_CHANGED BIT(2) + +static int unic_ets_prio_tc_validate(struct unic_dev *unic_dev, + struct ieee_ets *ets, u8 *changed, + u8 *vl_num) +{ + struct auxiliary_device *adev = unic_dev->comdev.adev; + struct ubase_caps *caps = ubase_get_dev_caps(adev); + u32 max_queue = unic_channels_max_num(adev); + u8 i, max_vl = 0; + + for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) { + if (ets->prio_tc[i] != unic_dev->channels.vl.prio_vl[i]) + *changed |= UNIC_PRIO_VL_MAP_CHANGED; + + max_vl = max(max_vl, ets->prio_tc[i] + 1); + } + + if (max_vl > caps->vl_num) { + unic_err(unic_dev, "tc num(%u) can't exceed max tc(%u).\n", + max_vl, caps->vl_num); + return -EINVAL; + } + + if (unic_get_rss_vl_num(unic_dev, max_vl) > max_queue) { + unic_err(unic_dev, + "tc num can't exceed queue num(%u).\n", max_queue); + return -EINVAL; + } + + *vl_num = max_vl; + + return 0; +} + +static int unic_ets_tc_bw_validate(struct unic_dev *unic_dev, + struct ieee_ets *ets, u8 *changed) +{ + struct auxiliary_device *adev = unic_dev->comdev.adev; + struct ubase_caps *caps = ubase_get_dev_caps(adev); + struct unic_vl *vl = &unic_dev->channels.vl; + /* continuous bitmap configured by the dcb tool */ + u16 tc_bitmap = (1 << caps->vl_num) - 1; + int ret; + u8 i; + + ret = ubase_check_qos_sch_param(adev, tc_bitmap, ets->tc_tx_bw, + ets->tc_tsa, false); + if (ret) + return ret; + + for (i = 0; i < caps->vl_num; i++) { + if (vl->vl_tsa[i] != ets->tc_tsa[i]) + *changed |= UNIC_TC_TSA_CHANGED; + + if (vl->vl_bw[i] != ets->tc_tx_bw[i]) + *changed |= UNIC_TC_BW_CHANGED; + } + + return 0; +} + +static int unic_setets_params_validate(struct unic_dev *unic_dev, + struct ieee_ets *ets, u8 *changed, + u8 *vl_num) +{ + int ret; + + ret = unic_ets_prio_tc_validate(unic_dev, ets, changed, vl_num); + if (ret) + return ret; + + return unic_ets_tc_bw_validate(unic_dev, ets, changed); +} + +static int unic_dcbnl_ieee_getets(struct net_device *ndev, struct ieee_ets *ets) +{ + struct unic_dev *unic_dev = netdev_priv(ndev); + struct auxiliary_device *adev = unic_dev->comdev.adev; + struct ubase_caps *caps = ubase_get_dev_caps(adev); + struct unic_vl *vl = &unic_dev->channels.vl; + + if (unic_resetting(ndev)) + return -EBUSY; + + if (!unic_dev_ets_supported(unic_dev)) + return -EOPNOTSUPP; + + memset(ets, 0, sizeof(*ets)); + ets->willing = 1; + ets->ets_cap = caps->vl_num; + + memcpy(ets->prio_tc, vl->prio_vl, sizeof(ets->prio_tc)); + memcpy(ets->tc_tx_bw, vl->vl_bw, sizeof(ets->tc_tx_bw)); + memcpy(ets->tc_tsa, vl->vl_tsa, sizeof(ets->tc_tsa)); + + return 0; +} + +static int unic_setets_preconditions(struct net_device *net_dev) +{ + struct unic_dev *unic_dev = netdev_priv(net_dev); + + if (!unic_dev_ets_supported(unic_dev)) + return -EOPNOTSUPP; + + if (netif_running(net_dev)) { + unic_err(unic_dev, + "failed to set ets, due to network interface is up, pls down it first and try again.\n"); + return -EBUSY; + } + + if (!(unic_dev->dcbx_cap & DCB_CAP_DCBX_VER_IEEE)) + return -EINVAL; + + if (unic_resetting(net_dev)) + return -EBUSY; + + return 0; +} + +static int unic_handle_prio_vl_change(struct unic_dev *unic_dev, + struct ieee_ets *ets, u8 changed) +{ + struct unic_vl *vl = &unic_dev->channels.vl; + u8 map_type; + int ret; + + if (!(changed & UNIC_PRIO_VL_MAP_CHANGED)) + return 0; + + map_type = vl->dscp_app_cnt ? UNIC_DSCP_VL_MAP : UNIC_PRIO_VL_MAP; + ret = unic_set_vl_map(unic_dev, vl->dscp_prio, ets->prio_tc, + map_type); + if (ret) + return ret; + + memcpy(vl->prio_vl, ets->prio_tc, sizeof(ets->prio_tc)); + + return 0; +} + +static inline void unic_convert_vl_sch_bw(struct ubase_caps *caps, u8 *vl_bw, + struct ieee_ets *ets) +{ + u8 i; + + for (i = 0; i < caps->vl_num; i++) { + vl_bw[caps->req_vl[i]] = ets->tc_tx_bw[i]; + vl_bw[caps->resp_vl[i]] = ets->tc_tx_bw[i]; + } +} + +static int unic_handle_tm_vl_sch_change(struct unic_dev *unic_dev, + struct ubase_caps *caps, + struct ieee_ets *ets) +{ + struct auxiliary_device *adev = unic_dev->comdev.adev; + struct unic_vl *vl = &unic_dev->channels.vl; + u8 vl_tsa[UBASE_MAX_VL_NUM] = {0}; + u8 vl_bw[UBASE_MAX_VL_NUM] = {0}; + u32 i; + + unic_convert_vl_sch_bw(caps, vl_bw, ets); + + for (i = 0; i < caps->vl_num; i++) { + if (ets->tc_tsa[i]) { + vl_tsa[caps->req_vl[i]] = ets->tc_tsa[i]; + vl_tsa[caps->resp_vl[i]] = ets->tc_tsa[i]; + } + } + + return ubase_config_tm_vl_sch(adev, vl->vl_bitmap, vl_bw, vl_tsa); +} + +static int unic_handle_vl_tsa_bw_change(struct unic_dev *unic_dev, + struct ieee_ets *ets, u8 changed) +{ + u8 *vl_tsa = unic_dev->channels.vl.vl_tsa; + u8 *vl_bw = unic_dev->channels.vl.vl_bw; + int ret; + + struct ubase_caps *caps = ubase_get_dev_caps(unic_dev->comdev.adev); + + if (!(changed & UNIC_TC_TSA_CHANGED || changed & UNIC_TC_BW_CHANGED)) + return 0; + + ret = unic_handle_tm_vl_sch_change(unic_dev, caps, ets); + if (ret) + return ret; + + memcpy(vl_tsa, ets->tc_tsa, sizeof(ets->tc_tsa)); + memcpy(vl_bw, ets->tc_tx_bw, sizeof(ets->tc_tx_bw)); + + return 0; +} + +static int unic_setets_config(struct net_device *ndev, struct ieee_ets *ets, + u8 changed, u8 vl_num) +{ + struct unic_dev *unic_dev = netdev_priv(ndev); + int ret; + + ret = unic_handle_prio_vl_change(unic_dev, ets, changed); + if (ret) + return ret; + + ret = unic_handle_vl_tsa_bw_change(unic_dev, ets, changed); + if (ret) + return ret; + + unic_dev->channels.vl.vl_num = vl_num; + if (unic_rss_vl_num_changed(unic_dev, vl_num)) + return unic_update_channels(unic_dev, vl_num); + + return 0; +} + +static int unic_dcbnl_ieee_setets(struct net_device *ndev, struct ieee_ets *ets) +{ + struct unic_dev *unic_dev = netdev_priv(ndev); + u8 changed = 0; + u8 vl_num = 0; + int ret; + + ret = unic_setets_preconditions(ndev); + if (ret) + return ret; + + ret = unic_setets_params_validate(unic_dev, ets, &changed, &vl_num); + if (ret) + return ret; + + if (!changed) + return 0; + + return unic_setets_config(ndev, ets, changed, vl_num); +} + +static int unic_dscp_prio_check(struct net_device *netdev, struct dcb_app *app) +{ + struct unic_dev *unic_dev = netdev_priv(netdev); + + if (!unic_dev_ets_supported(unic_dev)) + return -EOPNOTSUPP; + + if (netif_running(netdev)) { + unic_err(unic_dev, + "failed to set dscp-prio, due to network interface is up, pls down it first and try again.\n"); + return -EBUSY; + } + + if (app->selector != IEEE_8021QAZ_APP_SEL_DSCP || + app->protocol >= UBASE_MAX_DSCP || + app->priority >= UNIC_MAX_PRIO_NUM) + return -EINVAL; + + if (unic_resetting(netdev)) + return -EBUSY; + + return 0; +} + +static int unic_set_app(struct net_device *netdev, struct dcb_app *app, + struct unic_dev *unic_dev, struct unic_vl *vl) +{ + struct dcb_app old_app; + int ret; + + unic_info(unic_dev, "setapp dscp = %u, priority = %u.\n", + app->protocol, app->priority); + + ret = dcb_ieee_setapp(netdev, app); + if (ret) { + unic_err(unic_dev, "failed to add app, ret = %d.\n", ret); + return ret; + } + + old_app.selector = IEEE_8021QAZ_APP_SEL_DSCP; + old_app.protocol = app->protocol; + old_app.priority = vl->dscp_prio[app->protocol]; + + vl->dscp_prio[app->protocol] = app->priority; + ret = unic_set_vl_map(unic_dev, vl->dscp_prio, vl->prio_vl, + UNIC_DSCP_VL_MAP); + if (ret) { + vl->dscp_prio[app->protocol] = old_app.priority; + dcb_ieee_delapp(netdev, app); + return ret; + } + + if (old_app.priority == UNIC_INVALID_PRIORITY) { + vl->dscp_app_cnt++; + } else { + ret = dcb_ieee_delapp(netdev, &old_app); + if (ret) + unic_err(unic_dev, + "failed to delete old app, ret = %d.\n", ret); + } + + return ret; +} + +static int unic_dcbnl_ieee_setapp(struct net_device *netdev, + struct dcb_app *app) +{ + struct unic_dev *unic_dev = netdev_priv(netdev); + struct unic_vl *vl = &unic_dev->channels.vl; + int ret; + + ret = unic_dscp_prio_check(netdev, app); + if (ret) { + unic_err(unic_dev, "failed to set dscp-prio, ret = %d\n", ret); + return ret; + } + + /* dscp-prio already set */ + if (vl->dscp_prio[app->protocol] == app->priority) + return 0; + + return unic_set_app(netdev, app, unic_dev, vl); +} + +static int unic_del_app(struct net_device *netdev, struct dcb_app *app, + struct unic_dev *unic_dev, struct unic_vl *vl) +{ + u8 map_type = UNIC_DSCP_VL_MAP; + int ret; + + unic_info(unic_dev, "delapp dscp = %u, priority = %u\n", + app->protocol, app->priority); + + ret = dcb_ieee_delapp(netdev, app); + if (ret) + return ret; + + if (vl->dscp_app_cnt <= 1) + map_type = UNIC_PRIO_VL_MAP; + + vl->dscp_prio[app->protocol] = UNIC_INVALID_PRIORITY; + ret = unic_set_vl_map(unic_dev, vl->dscp_prio, vl->prio_vl, + map_type); + if (ret) { + vl->dscp_prio[app->protocol] = app->priority; + dcb_ieee_setapp(netdev, app); + return ret; + } + + if (vl->dscp_app_cnt) + vl->dscp_app_cnt--; + + return 0; +} + +static int unic_dcbnl_ieee_delapp(struct net_device *netdev, + struct dcb_app *app) +{ + struct unic_dev *unic_dev = netdev_priv(netdev); + struct unic_vl *vl = &unic_dev->channels.vl; + int ret; + + ret = unic_dscp_prio_check(netdev, app); + if (ret) { + unic_err(unic_dev, "failed to del dscp-prio, ret = %d.\n", ret); + return ret; + } + + if (app->priority != vl->dscp_prio[app->protocol]) { + unic_err(unic_dev, "failed to del no match dscp-prio.\n"); + return -EINVAL; + } + + return unic_del_app(netdev, app, unic_dev, vl); +} + +static u8 unic_dcbnl_getdcbx(struct net_device *ndev) +{ + struct unic_dev *unic_dev = netdev_priv(ndev); + + return unic_dev->dcbx_cap; +} + +static u8 unic_dcbnl_setdcbx(struct net_device *ndev, u8 mode) +{ + struct unic_dev *unic_dev = netdev_priv(ndev); + + if ((mode & DCB_CAP_DCBX_LLD_MANAGED) || + (mode & DCB_CAP_DCBX_VER_CEE) || + !(mode & DCB_CAP_DCBX_HOST)) + return 1; + + unic_info(unic_dev, "set dcbx mode = %u\n.", mode); + + unic_dev->dcbx_cap = mode; + + return 0; +} + +static int unic_ieee_getmaxrate(struct net_device *ndev, + struct ieee_maxrate *maxrate) +{ + struct unic_dev *unic_dev = netdev_priv(ndev); + + if (!unic_dev_ets_supported(unic_dev) || + !unic_dev_tc_speed_limit_supported(unic_dev)) + return -EOPNOTSUPP; + + memcpy(maxrate->tc_maxrate, unic_dev->channels.vl.vl_maxrate, + sizeof(maxrate->tc_maxrate)); + return 0; +} + +static int unic_check_maxrate(struct unic_dev *unic_dev, + struct ieee_maxrate *maxrate) +{ + u32 max_speed = unic_dev->hw.mac.max_speed; + int i; + + for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) { + if (!maxrate->tc_maxrate[i]) + continue; + + if (maxrate->tc_maxrate[i] / UNIC_MBYTE_PER_SEND > max_speed || + maxrate->tc_maxrate[i] < UNIC_MBYTE_PER_SEND) { + unic_err(unic_dev, + "invalid max_rate(%llubit), the range is [1Mbit, %uMbit].\n", + maxrate->tc_maxrate[i] * BITS_PER_BYTE, + max_speed); + return -EINVAL; + } + } + + return 0; +} + +static int unic_ieee_setmaxrate(struct net_device *ndev, + struct ieee_maxrate *maxrate) +{ + struct unic_dev *unic_dev = netdev_priv(ndev); + struct unic_vl *vl = &unic_dev->channels.vl; + int ret; + + if (!unic_dev_ets_supported(unic_dev) || + !unic_dev_tc_speed_limit_supported(unic_dev)) + return -EOPNOTSUPP; + + if (!(unic_dev->dcbx_cap & DCB_CAP_DCBX_VER_IEEE)) + return -EINVAL; + + ret = unic_check_maxrate(unic_dev, maxrate); + if (ret) + return ret; + + ret = unic_config_vl_rate_limit(unic_dev, maxrate->tc_maxrate, + vl->vl_bitmap); + if (ret) + return ret; + + memcpy(vl->vl_maxrate, maxrate->tc_maxrate, + sizeof(maxrate->tc_maxrate)); + + return 0; +} + static const struct dcbnl_rtnl_ops unic_dcbnl_ops = { + .ieee_getets = unic_dcbnl_ieee_getets, + .ieee_setets = unic_dcbnl_ieee_setets, + .ieee_getmaxrate = unic_ieee_getmaxrate, + .ieee_setmaxrate = unic_ieee_setmaxrate, + .ieee_setapp = unic_dcbnl_ieee_setapp, + .ieee_delapp = unic_dcbnl_ieee_delapp, + .getdcbx = unic_dcbnl_getdcbx, + .setdcbx = unic_dcbnl_setdcbx, }; void unic_set_dcbnl_ops(struct net_device *netdev) diff --git a/drivers/net/ub/unic/unic_dev.c b/drivers/net/ub/unic/unic_dev.c index a0ea8edee2efbdec6db6104ffe402cd472da8b3a..f8d5676bfc1f2c8b8ee81124312801bf07bb3f70 100644 --- a/drivers/net/ub/unic/unic_dev.c +++ b/drivers/net/ub/unic/unic_dev.c @@ -248,10 +248,12 @@ static int unic_init_vl_info(struct unic_dev *unic_dev) return ret; ret = unic_init_vl_maxrate(unic_dev); - if (ret) + if (ret && ret != -EPERM) return ret; - return unic_init_vl_sch(unic_dev); + ret = unic_init_vl_sch(unic_dev); + + return ret == -EPERM ? 0 : ret; } static int unic_init_channels_attr(struct unic_dev *unic_dev) @@ -293,6 +295,30 @@ static void unic_uninit_channels_attr(struct unic_dev *unic_dev) mutex_destroy(&channels->mutex); } +u16 unic_cqe_period_round_down(u16 cqe_period) +{ + u16 period[] = { + UNIC_CQE_PERIOD_0, + UNIC_CQE_PERIOD_4, + UNIC_CQE_PERIOD_16, + UNIC_CQE_PERIOD_64, + UNIC_CQE_PERIOD_256, + UNIC_CQE_PERIOD_1024, + UNIC_CQE_PERIOD_4096, + UNIC_CQE_PERIOD_16384, + UNIC_CQE_PERIOD_ERR + }; + u16 i; + + for (i = 0; i < ARRAY_SIZE(period) - 1; i++) { + if (cqe_period >= period[i] && + cqe_period < period[i + 1]) + return period[i]; + } + + return UNIC_CQE_PERIOD_ERR; +} + int unic_init_tx(struct unic_dev *unic_dev, u32 num) { struct unic_channel *c; @@ -535,21 +561,17 @@ static int unic_dev_init_mtu(struct unic_dev *unic_dev) { struct net_device *netdev = unic_dev->comdev.netdev; struct unic_caps *caps = &unic_dev->caps; - int ret; netdev->mtu = UB_DATA_LEN; netdev->max_mtu = caps->max_trans_unit; netdev->min_mtu = caps->min_trans_unit; - ret = unic_config_mtu(unic_dev, netdev->mtu); - if (ret == -EPERM) - return 0; - - return ret; + return unic_config_mtu(unic_dev, netdev->mtu); } static int unic_init_mac(struct unic_dev *unic_dev) { + struct unic_link_stats *record = &unic_dev->stats.link_record; struct unic_mac *mac = &unic_dev->hw.mac; int ret; @@ -558,11 +580,11 @@ static int unic_init_mac(struct unic_dev *unic_dev) ret = unic_set_mac_speed_duplex(unic_dev, mac->speed, mac->duplex, mac->lanes); - if (ret && ret != -EPERM) + if (ret) return ret; ret = unic_set_mac_autoneg(unic_dev, mac->autoneg); - if (ret && ret != -EPERM) + if (ret) return ret; ret = unic_dev_fec_supported(unic_dev) && mac->user_fec_mode ? @@ -577,9 +599,17 @@ static int unic_init_mac(struct unic_dev *unic_dev) return ret; } + mutex_init(&record->lock); return 0; } +static void unic_uninit_mac(struct unic_dev *unic_dev) +{ + struct unic_link_stats *record = &unic_dev->stats.link_record; + + mutex_destroy(&record->lock); +} + int unic_set_mtu(struct unic_dev *unic_dev, int new_mtu) { u16 max_frame_size; @@ -588,9 +618,7 @@ int unic_set_mtu(struct unic_dev *unic_dev, int new_mtu) new_mtu = max(new_mtu, UB_DATA_LEN); ret = unic_check_validate_dump_mtu(unic_dev, new_mtu, &max_frame_size); - if (ret == -EPERM) { - return 0; - } else if (ret < 0) { + if (ret) { unic_err(unic_dev, "invalid MTU(%d), please check, ret = %d.\n", new_mtu, ret); return -EINVAL; @@ -798,11 +826,11 @@ static int unic_init_netdev_priv(struct net_device *netdev, ret = unic_init_dev_addr(priv); if (ret) - goto err_uninit_vport; + goto unic_unint_mac; ret = unic_init_channels_attr(priv); if (ret) - goto err_uninit_vport; + goto unic_unint_mac; ret = unic_init_channels(priv, priv->channels.num); if (ret) { @@ -816,6 +844,8 @@ static int unic_init_netdev_priv(struct net_device *netdev, err_uninit_channels_attr: unic_uninit_channels_attr(priv); +unic_unint_mac: + unic_uninit_mac(priv); err_uninit_vport: unic_uninit_vport(priv); destroy_lock: @@ -830,6 +860,7 @@ static void unic_uninit_netdev_priv(struct net_device *netdev) unic_uninit_channels(priv); unic_uninit_channels_attr(priv); + unic_uninit_mac(priv); unic_uninit_vport(priv); mutex_destroy(&priv->act_info.mutex); } @@ -869,6 +900,13 @@ void unic_remove_period_task(struct unic_dev *unic_dev) cancel_delayed_work_sync(&unic_dev->service_task); } +bool unic_rss_vl_num_changed(struct unic_dev *unic_dev, u8 vl_num) +{ + struct unic_channels *channels = &unic_dev->channels; + + return channels->rss_vl_num != unic_get_rss_vl_num(unic_dev, vl_num); +} + int unic_change_rss_size(struct unic_dev *unic_dev, u32 new_rss_size, u32 org_rss_size) { @@ -896,6 +934,21 @@ int unic_change_rss_size(struct unic_dev *unic_dev, u32 new_rss_size, return ret; } +int unic_update_channels(struct unic_dev *unic_dev, u8 vl_num) +{ + struct auxiliary_device *adev = unic_dev->comdev.adev; + struct unic_channels *channels = &unic_dev->channels; + u32 new_rss_size, old_rss_size = channels->rss_size; + + channels->rss_vl_num = unic_get_rss_vl_num(unic_dev, vl_num); + if (old_rss_size * channels->rss_vl_num > unic_channels_max_num(adev)) + new_rss_size = unic_get_max_rss_size(unic_dev); + else + new_rss_size = old_rss_size; + + return unic_change_rss_size(unic_dev, new_rss_size, old_rss_size); +} + static struct net_device *unic_alloc_netdev(struct auxiliary_device *adev) { struct ubase_caps *caps = ubase_get_dev_caps(adev); diff --git a/drivers/net/ub/unic/unic_dev.h b/drivers/net/ub/unic/unic_dev.h index f0df795ad011f879b248822a3afefd2be5668a72..51708850e38d8791ca596555ea62292607495cdc 100644 --- a/drivers/net/ub/unic/unic_dev.h +++ b/drivers/net/ub/unic/unic_dev.h @@ -182,8 +182,20 @@ struct unic_fec_stats { struct unic_fec_stats_item lane[UNIC_FEC_STATS_MAX_LANE]; }; +#define LINK_STAT_MAX_IDX 10U +struct unic_link_stats { + u64 link_up_cnt; + u64 link_down_cnt; + struct { + bool link_status; + time64_t link_tv_sec; + } stats[LINK_STAT_MAX_IDX]; + struct mutex lock; /* protects link record */ +}; + struct unic_stats { struct unic_fec_stats fec_stats; + struct unic_link_stats link_record; }; struct unic_addr_tbl { @@ -245,14 +257,18 @@ int unic_init_channels(struct unic_dev *unic_dev, u32 channels_num); void unic_uninit_channels(struct unic_dev *unic_dev); void unic_start_period_task(struct net_device *netdev); void unic_remove_period_task(struct unic_dev *unic_dev); +void unic_update_queue_info(struct unic_dev *unic_dev); int unic_init_wq(void); void unic_destroy_wq(void); +u16 unic_cqe_period_round_down(u16 cqe_period); int unic_init_rx(struct unic_dev *unic_dev, u32 num); int unic_init_tx(struct unic_dev *unic_dev, u32 num); void unic_destroy_rx(struct unic_dev *unic_dev, u32 num); void unic_destroy_tx(struct unic_dev *unic_dev, u32 num); +bool unic_rss_vl_num_changed(struct unic_dev *unic_dev, u8 vl_num); int unic_change_rss_size(struct unic_dev *unic_dev, u32 new_rss_size, u32 org_rss_size); +int unic_update_channels(struct unic_dev *unic_dev, u8 vl_num); int unic_set_vl_map(struct unic_dev *unic_dev, u8 *dscp_prio, u8 *prio_vl, u8 map_type); int unic_dbg_log(void); @@ -262,6 +278,11 @@ static inline bool unic_dev_ubl_supported(struct unic_dev *unic_dev) return ubase_adev_ubl_supported(unic_dev->comdev.adev); } +static inline bool unic_dev_eth_mac_supported(struct unic_dev *unic_dev) +{ + return ubase_adev_eth_mac_supported(unic_dev->comdev.adev); +} + static inline bool unic_dev_ets_supported(struct unic_dev *unic_dev) { return unic_get_cap_bit(unic_dev, UNIC_SUPPORT_ETS_B); @@ -272,11 +293,6 @@ static inline bool unic_dev_fec_supported(struct unic_dev *unic_dev) return unic_get_cap_bit(unic_dev, UNIC_SUPPORT_FEC_B); } -static inline bool unic_dev_rss_supported(struct unic_dev *unic_dev) -{ - return unic_get_cap_bit(unic_dev, UNIC_SUPPORT_RSS_B); -} - static inline bool unic_dev_tc_speed_limit_supported(struct unic_dev *unic_dev) { return unic_get_cap_bit(unic_dev, UNIC_SUPPORT_TC_SPEED_LIMIT_B); @@ -327,6 +343,27 @@ static inline bool unic_is_initing_or_resetting(struct unic_dev *unic_dev) return __unic_resetting(unic_dev) || __unic_initing(unic_dev); } +static inline u32 unic_read_reg(struct unic_dev *unic_dev, u32 reg) +{ + struct ubase_resource_space *io_base = ubase_get_io_base(unic_dev->comdev.adev); + u8 __iomem *reg_addr; + + if (!io_base) + return 0; + + reg_addr = READ_ONCE(io_base->addr); + return readl(reg_addr + reg); +} + +static inline u8 unic_get_rss_vl_num(struct unic_dev *unic_dev, u8 max_vl) +{ + struct auxiliary_device *adev = unic_dev->comdev.adev; + struct ubase_adev_qos *qos = ubase_get_adev_qos(adev); + u8 vl_num = min(UNIC_RSS_MAX_VL_NUM, qos->nic_vl_num); + + return max_vl < vl_num ? max_vl : vl_num; +} + static inline u32 unic_get_sq_cqe_mask(struct unic_dev *unic_dev) { return unic_dev->channels.sq_cqe_depth - 1; diff --git a/drivers/net/ub/unic/unic_ethtool.c b/drivers/net/ub/unic/unic_ethtool.c index 9e50f0332d4e6759c74b8b243ad1e08371d7f30d..f2cfa3df1126095085b770f1550cbaa24264fc48 100644 --- a/drivers/net/ub/unic/unic_ethtool.c +++ b/drivers/net/ub/unic/unic_ethtool.c @@ -24,6 +24,46 @@ static u32 unic_get_link_status(struct net_device *netdev) return unic_dev->sw_link_status; } +static void unic_get_port_type(struct unic_dev *unic_dev, + struct ethtool_link_ksettings *cmd) +{ + u8 module_type = unic_dev->hw.mac.module_type; + u8 media_type = unic_dev->hw.mac.media_type; + + switch (media_type) { + case UNIC_MEDIA_TYPE_NONE: + case UNIC_MEDIA_TYPE_BACKPLANE: + cmd->base.port = PORT_NONE; + break; + case UNIC_MEDIA_TYPE_FIBER: + if (module_type == UNIC_MODULE_TYPE_CR) + cmd->base.port = PORT_DA; + else + cmd->base.port = PORT_FIBRE; + break; + default: + cmd->base.port = PORT_NONE; + break; + } +} + +static void unic_get_ksettings(struct unic_dev *unic_dev, + struct ethtool_link_ksettings *cmd) +{ + struct unic_mac *mac = &unic_dev->hw.mac; + + unic_get_port_type(unic_dev, cmd); + + cmd->base.speed = mac->speed; + cmd->base.duplex = mac->duplex; + cmd->base.autoneg = mac->autoneg; + + linkmode_copy(cmd->link_modes.supported, mac->supported); + linkmode_copy(cmd->link_modes.advertising, mac->advertising); + + cmd->lanes = mac->lanes; +} + static int unic_get_link_ksettings(struct net_device *netdev, struct ethtool_link_ksettings *cmd) { @@ -32,6 +72,13 @@ static int unic_get_link_ksettings(struct net_device *netdev, /* Ensure that the latest information is obtained. */ unic_update_port_info(unic_dev); + unic_get_ksettings(unic_dev, cmd); + + if (!unic_get_link_status(netdev)) { + cmd->base.speed = SPEED_UNKNOWN; + cmd->base.duplex = DUPLEX_UNKNOWN; + } + return 0; } @@ -67,9 +114,6 @@ static int unic_get_fecparam(struct net_device *ndev, struct unic_dev *unic_dev = netdev_priv(ndev); struct unic_mac *mac = &unic_dev->hw.mac; - if (!unic_dev_fec_supported(unic_dev)) - return -EOPNOTSUPP; - fec->fec = mac->fec_ability; fec->active_fec = mac->fec_mode; @@ -109,6 +153,198 @@ static int unic_set_fecparam(struct net_device *ndev, return 0; } +static int unic_get_coalesce(struct net_device *netdev, + struct ethtool_coalesce *cmd, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct unic_dev *unic_dev = netdev_priv(netdev); + struct unic_coalesce *tx_coal = &unic_dev->channels.unic_coal.tx_coal; + struct unic_coalesce *rx_coal = &unic_dev->channels.unic_coal.rx_coal; + + if (unic_resetting(netdev)) + return -EBUSY; + + cmd->tx_coalesce_usecs = tx_coal->int_gl; + cmd->rx_coalesce_usecs = rx_coal->int_gl; + + cmd->tx_max_coalesced_frames = tx_coal->int_ql; + cmd->rx_max_coalesced_frames = rx_coal->int_ql; + + return 0; +} + +static int unic_check_gl_coalesce_para(struct net_device *netdev, + struct ethtool_coalesce *cmd) +{ + struct unic_dev *unic_dev = netdev_priv(netdev); + u32 rx_gl, tx_gl; + + if (cmd->rx_coalesce_usecs > unic_dev->caps.max_int_gl) { + unic_err(unic_dev, + "invalid rx-usecs value, rx-usecs range is [0, %u].\n", + unic_dev->caps.max_int_gl); + return -EINVAL; + } + + if (cmd->tx_coalesce_usecs > unic_dev->caps.max_int_gl) { + unic_err(unic_dev, + "invalid tx-usecs value, tx-usecs range is [0, %u].\n", + unic_dev->caps.max_int_gl); + return -EINVAL; + } + + rx_gl = unic_cqe_period_round_down(cmd->rx_coalesce_usecs); + if (rx_gl != cmd->rx_coalesce_usecs) { + unic_err(unic_dev, + "invalid rx_usecs(%u), because it must be power of 4.\n", + cmd->rx_coalesce_usecs); + return -EINVAL; + } + + tx_gl = unic_cqe_period_round_down(cmd->tx_coalesce_usecs); + if (tx_gl != cmd->tx_coalesce_usecs) { + unic_err(unic_dev, + "invalid tx_usecs(%u), because it must be power of 4.\n", + cmd->tx_coalesce_usecs); + return -EINVAL; + } + + return 0; +} + +static int unic_check_ql_coalesce_para(struct net_device *netdev, + struct ethtool_coalesce *cmd) +{ + struct unic_dev *unic_dev = netdev_priv(netdev); + + if ((cmd->tx_max_coalesced_frames || cmd->rx_max_coalesced_frames) && + !unic_dev->caps.max_int_ql) { + unic_err(unic_dev, "coalesced frames is not supported.\n"); + return -EOPNOTSUPP; + } + + if (cmd->tx_max_coalesced_frames > unic_dev->caps.max_int_ql || + cmd->rx_max_coalesced_frames > unic_dev->caps.max_int_ql) { + unic_err(unic_dev, + "invalid coalesced frames value, range is [0, %u].\n", + unic_dev->caps.max_int_ql); + return -ERANGE; + } + + return 0; +} + +static int +unic_check_coalesce_para(struct net_device *netdev, + struct ethtool_coalesce *cmd, + struct kernel_ethtool_coalesce *kernel_coal) +{ + struct unic_dev *unic_dev = netdev_priv(netdev); + int ret; + + if (cmd->use_adaptive_rx_coalesce || cmd->use_adaptive_tx_coalesce) { + unic_err(unic_dev, + "not support to enable adaptive coalesce.\n"); + return -EINVAL; + } + + ret = unic_check_gl_coalesce_para(netdev, cmd); + if (ret) { + unic_err(unic_dev, + "failed to check gl coalesce param, ret = %d.\n", ret); + return ret; + } + + ret = unic_check_ql_coalesce_para(netdev, cmd); + if (ret) + unic_err(unic_dev, + "failed to check ql coalesce param, ret = %d.\n", ret); + + return ret; +} + +static int unic_set_coalesce(struct net_device *netdev, + struct ethtool_coalesce *cmd, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct unic_dev *unic_dev = netdev_priv(netdev); + struct unic_coal_txrx *unic_coal = &unic_dev->channels.unic_coal; + struct unic_coalesce *tx_coal = &unic_coal->tx_coal; + struct unic_coalesce *rx_coal = &unic_coal->rx_coal; + struct unic_coalesce old_tx_coal, old_rx_coal; + int ret, ret1; + + if (unic_resetting(netdev)) + return -EBUSY; + + ret = unic_check_coalesce_para(netdev, cmd, kernel_coal); + if (ret) + return ret; + + memcpy(&old_tx_coal, tx_coal, sizeof(struct unic_coalesce)); + memcpy(&old_rx_coal, rx_coal, sizeof(struct unic_coalesce)); + + tx_coal->int_gl = cmd->tx_coalesce_usecs; + rx_coal->int_gl = cmd->rx_coalesce_usecs; + + tx_coal->int_ql = cmd->tx_max_coalesced_frames; + rx_coal->int_ql = cmd->rx_max_coalesced_frames; + + unic_uninit_channels(unic_dev); + + ret = unic_init_channels(unic_dev, unic_dev->channels.num); + if (ret) { + netdev_err(netdev, "failed to init channels, ret = %d.\n", ret); + memcpy(tx_coal, &old_tx_coal, sizeof(struct unic_coalesce)); + memcpy(rx_coal, &old_rx_coal, sizeof(struct unic_coalesce)); + ret1 = unic_init_channels(unic_dev, unic_dev->channels.num); + if (ret1) + unic_err(unic_dev, + "failed to recover old channels, ret = %d.\n", + ret1); + } + + return ret; +} + +static const struct unic_reset_type_map unic_ethtool_reset_map[] = { + {ETH_RESET_DEDICATED, UBASE_UE_RESET}, +}; + +static int unic_reset(struct net_device *ndev, u32 *flags) +{ + enum ubase_reset_type reset_type = UBASE_NO_RESET; + struct unic_dev *unic_dev = netdev_priv(ndev); + enum ethtool_reset_flags reset_flags; + u32 i; + + if (unic_resetting(ndev)) { + unic_err(unic_dev, "failed to reset, due to dev resetting.\n"); + return -EBUSY; + } + + for (i = 0; i < ARRAY_SIZE(unic_ethtool_reset_map); i++) { + if (unic_ethtool_reset_map[i].reset_flags == *flags) { + reset_type = unic_ethtool_reset_map[i].reset_type; + reset_flags = unic_ethtool_reset_map[i].reset_flags; + break; + } + } + + if (reset_type == UBASE_NO_RESET) + return -EOPNOTSUPP; + + unic_info(unic_dev, + "ethtool setting reset type, type = %u.\n", reset_type); + + ubase_reset_event(unic_dev->comdev.adev, reset_type); + *flags &= ~reset_flags; + + return 0; +} + #define UNIC_ETHTOOL_RING (ETHTOOL_RING_USE_RX_BUF_LEN | \ ETHTOOL_RING_USE_TX_PUSH) #define UNIC_ETHTOOL_COALESCE (ETHTOOL_COALESCE_USECS | \ @@ -122,6 +358,11 @@ static const struct ethtool_ops unic_ethtool_ops = { .get_link = unic_get_link_status, .get_link_ksettings = unic_get_link_ksettings, .get_drvinfo = unic_get_driver_info, + .get_regs_len = unic_get_regs_len, + .get_regs = unic_get_regs, + .get_ethtool_stats = unic_get_stats, + .get_strings = unic_get_stats_strings, + .get_sset_count = unic_get_sset_count, .get_channels = unic_get_channels, .set_channels = unic_set_channels, .get_ringparam = unic_get_channels_param, @@ -129,6 +370,9 @@ static const struct ethtool_ops unic_ethtool_ops = { .get_fecparam = unic_get_fecparam, .set_fecparam = unic_set_fecparam, .get_fec_stats = unic_get_fec_stats, + .get_coalesce = unic_get_coalesce, + .set_coalesce = unic_set_coalesce, + .reset = unic_reset, }; void unic_set_ethtool_ops(struct net_device *netdev) diff --git a/drivers/net/ub/unic/unic_event.c b/drivers/net/ub/unic/unic_event.c index bcd1e210f447753bf9a587bf62c8f384b097a176..1785e0aad7f1e07f5acf82d64a58372527c4883f 100644 --- a/drivers/net/ub/unic/unic_event.c +++ b/drivers/net/ub/unic/unic_event.c @@ -23,6 +23,7 @@ #include "unic_netdev.h" #include "unic_qos_hw.h" #include "unic_rack_ip.h" +#include "unic_reset.h" #include "unic_event.h" int unic_comp_handler(struct notifier_block *nb, unsigned long jfcn, void *data) @@ -52,6 +53,124 @@ int unic_comp_handler(struct notifier_block *nb, unsigned long jfcn, void *data) return 0; } +static void unic_activate_event_process(struct unic_dev *unic_dev) +{ + struct unic_act_info *act_info = &unic_dev->act_info; + struct net_device *netdev = unic_dev->comdev.netdev; + int ret; + + if (test_bit(UNIC_STATE_DISABLED, &unic_dev->state)) { + unic_err(unic_dev, + "failed to process activate event, device is not ready.\n"); + return; + } + + rtnl_lock(); + + if (!test_bit(UNIC_STATE_DEACTIVATE, &unic_dev->state)) + goto unlock; + + /* if network interface has already been stopped, + * no need to open by activate event + */ + if (test_bit(UNIC_STATE_DOWN, &unic_dev->state)) + goto out; + + ret = unic_net_open_no_link_change(netdev); + if (ret) + unic_warn(unic_dev, "failed to open net, ret = %d.\n", ret); + + ret = unic_activate_promisc_mode(unic_dev, true); + if (ret) + unic_warn(unic_dev, "failed to open promisc, ret = %d.\n", ret); + else + clear_bit(UNIC_VPORT_STATE_PROMISC_CHANGE, &unic_dev->vport.state); + +out: + mutex_lock(&act_info->mutex); + act_info->deactivate = false; + mutex_unlock(&act_info->mutex); + clear_bit(UNIC_STATE_DEACTIVATE, &unic_dev->state); +unlock: + rtnl_unlock(); +} + +static void unic_deactivate_event_process(struct unic_dev *unic_dev) +{ + struct unic_act_info *act_info = &unic_dev->act_info; + struct net_device *netdev = unic_dev->comdev.netdev; + int ret; + + if (test_bit(UNIC_STATE_DISABLED, &unic_dev->state)) { + unic_err(unic_dev, + "failed to process deactivate event, device is not ready.\n"); + return; + } + + rtnl_lock(); + + if (test_bit(UNIC_STATE_DEACTIVATE, &unic_dev->state)) + goto unlock; + + /* when deactivate event occurs, set flag to true to prevent + * periodic tasks changing promisc + */ + mutex_lock(&act_info->mutex); + act_info->deactivate = true; + mutex_unlock(&act_info->mutex); + + ret = unic_activate_promisc_mode(unic_dev, false); + if (ret) + unic_warn(unic_dev, "failed to close promisc, ret = %d.\n", ret); + else + set_bit(UNIC_VPORT_STATE_PROMISC_CHANGE, &unic_dev->vport.state); + + /* if network interface has already been stopped, + * no need to stop again by deactivate event + */ + if (test_bit(UNIC_STATE_DOWN, &unic_dev->state)) + goto out; + + unic_net_stop_no_link_change(netdev); + +out: + set_bit(UNIC_STATE_DEACTIVATE, &unic_dev->state); +unlock: + rtnl_unlock(); +} + +static void unic_activate_handler(struct auxiliary_device *adev, bool activate) +{ + struct unic_dev *unic_dev = dev_get_drvdata(&adev->dev); + + unic_info(unic_dev, "receive %s event callback.\n", + activate ? "activate" : "deactivate"); + + if (activate) + unic_activate_event_process(unic_dev); + else + unic_deactivate_event_process(unic_dev); +} + +static void unic_rack_port_reset(struct unic_dev *unic_dev, bool link_up) +{ + if (link_up) + unic_dev->hw.mac.link_status = UNIC_LINK_STATUS_UP; + else + unic_dev->hw.mac.link_status = UNIC_LINK_STATUS_DOWN; +} + +static void unic_port_handler(struct auxiliary_device *adev, bool link_up) +{ + struct unic_dev *unic_dev = dev_get_drvdata(&adev->dev); + struct net_device *netdev = unic_dev->comdev.netdev; + + if (!netif_running(netdev)) + return; + + unic_rack_port_reset(unic_dev, link_up); +} + static struct ubase_ctrlq_event_nb unic_ctrlq_events[] = { { .service_type = UBASE_CTRLQ_SER_TYPE_IP_ACL, @@ -128,27 +247,105 @@ static int unic_register_crq_event(struct auxiliary_device *adev) return 0; } +static void unic_unregister_ae_event(struct auxiliary_device *adev, + u8 asyn_event_num) +{ + struct unic_dev *unic_dev = dev_get_drvdata(&adev->dev); + u8 i; + + for (i = 0; i < asyn_event_num; i++) + ubase_event_unregister(adev, &unic_dev->ae_nbs[i]); +} + +static int unic_ae_jetty_level_error(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct ubase_event_nb *ev_nb = container_of(nb, + struct ubase_event_nb, nb); + struct auxiliary_device *adev = (struct auxiliary_device *)ev_nb->back; + struct unic_dev *unic_dev = dev_get_drvdata(&adev->dev); + struct ubase_aeq_notify_info *info = data; + u32 queue_num; + + /* Normally, UNIC does not report such abnormal events, + * but in order to maintain its scalability, + * unic reserves the reset processing of such events. + */ + queue_num = info->aeqe->event.queue_event.num; + unic_err(unic_dev, + "recv jetty level error, event_type = 0x%x, sub_type = 0x%x, queue_num = %u.\n", + info->event_type, info->sub_type, queue_num); + + ubase_reset_event(adev, UBASE_UE_RESET); + + return 0; +} + +static int unic_register_ae_event(struct auxiliary_device *adev) +{ + struct ubase_event_nb unic_ae_nbs[UNIC_AE_LEVEL_NUM] = { + { + UBASE_DRV_UNIC, + UBASE_EVENT_TYPE_JETTY_LEVEL_ERROR, + { unic_ae_jetty_level_error }, + adev + }, + }; + struct unic_dev *unic_dev = dev_get_drvdata(&adev->dev); + int ret; + u8 i; + + for (i = 0; i < ARRAY_SIZE(unic_ae_nbs); i++) { + unic_dev->ae_nbs[i] = unic_ae_nbs[i]; + ret = ubase_event_register(adev, &unic_dev->ae_nbs[i]); + if (ret) { + dev_err(adev->dev.parent, + "failed to register asyn event[%u], ret = %d.\n", + unic_dev->ae_nbs[i].event_type, ret); + unic_unregister_ae_event(adev, i); + return ret; + } + } + + return ret; +} + int unic_register_event(struct auxiliary_device *adev) { int ret; - ret = unic_register_crq_event(adev); + ret = unic_register_ae_event(adev); if (ret) return ret; + ret = unic_register_crq_event(adev); + if (ret) + goto unregister_ae; + ret = unic_register_ctrlq_event(adev); if (ret) goto unregister_crq; + ubase_port_register(adev, unic_port_handler); + ubase_reset_register(adev, unic_reset_handler); + ubase_activate_register(adev, unic_activate_handler); + return 0; unregister_crq: unic_unregister_crq_event(adev, ARRAY_SIZE(unic_crq_events)); +unregister_ae: + unic_unregister_ae_event(adev, UNIC_AE_LEVEL_NUM); + return ret; } void unic_unregister_event(struct auxiliary_device *adev) { + ubase_activate_unregister(adev); + ubase_reset_unregister(adev); + ubase_port_unregister(adev); unic_unregister_ctrlq_event(adev, ARRAY_SIZE(unic_ctrlq_events)); unic_unregister_crq_event(adev, ARRAY_SIZE(unic_crq_events)); + unic_unregister_ae_event(adev, UNIC_AE_LEVEL_NUM); } diff --git a/drivers/net/ub/unic/unic_hw.c b/drivers/net/ub/unic/unic_hw.c index d2b7026514c460dc9f579b3b30e0665a71e70a28..565ac56fb638105c52c46d23a26ff7bae87e590d 100644 --- a/drivers/net/ub/unic/unic_hw.c +++ b/drivers/net/ub/unic/unic_hw.c @@ -76,7 +76,7 @@ int unic_set_mac_autoneg(struct unic_dev *unic_dev, u8 autoneg) sizeof(req), &req); ret = ubase_cmd_send_in(unic_dev->comdev.adev, &in); - if (ret && ret != -EPERM) + if (ret) dev_err(unic_dev->comdev.adev->dev.parent, "failed to send cmd in config autoneg(%u), ret = %d.\n", autoneg, ret); @@ -105,7 +105,7 @@ int unic_set_mac_speed_duplex(struct unic_dev *unic_dev, u32 speed, u8 duplex, sizeof(req), &req); ret = ubase_cmd_send_in(unic_dev->comdev.adev, &in); - if (ret && ret != -EPERM) + if (ret) dev_err(unic_dev->comdev.adev->dev.parent, "failed to send cmd in config speed(%u), ret = %d.\n", speed, ret); @@ -334,9 +334,7 @@ int unic_set_promisc_mode(struct unic_dev *unic_dev, time_out = unic_cmd_timeout(unic_dev); ret = ubase_cmd_send_in_ex(adev, &in, time_out); - if (ret == -EPERM) - return 0; - else if (ret) + if (ret) unic_err(unic_dev, "failed to set promisc mode, ret = %d.\n", ret); @@ -561,15 +559,15 @@ static int unic_query_flush_status(struct unic_dev *unic_dev, u8 *status) struct ubase_cmd_buf in; int ret; + if (unic_dev_ubl_supported(unic_dev)) + return -EOPNOTSUPP; + ubase_fill_inout_buf(&in, UBASE_OPC_QUERY_FLUSH_STATUS, true, 0, NULL); ubase_fill_inout_buf(&out, UBASE_OPC_QUERY_FLUSH_STATUS, false, sizeof(resp), &resp); ret = ubase_cmd_send_inout(unic_dev->comdev.adev, &in, &out); if (ret) { - if (ret == -EPERM) - return -EOPNOTSUPP; - unic_err(unic_dev, "failed to send cmd when query flush status, ret = %d.\n", ret); @@ -676,6 +674,29 @@ int unic_cfg_vport_buf(struct unic_dev *unic_dev, bool init) return ret; } +int unic_query_vport_ctx(struct unic_dev *unic_dev, u16 offset, + struct unic_vport_ctx_cmd *resp) +{ + struct unic_vport_ctx_cmd req; + struct ubase_cmd_buf in, out; + int ret; + + memset(&req, 0, sizeof(req)); + req.offset = cpu_to_le16(offset); + + ubase_fill_inout_buf(&in, UBASE_OPC_VPORT_CTX, true, + sizeof(req), &req); + ubase_fill_inout_buf(&out, UBASE_OPC_VPORT_CTX, true, + sizeof(*resp), resp); + + ret = ubase_cmd_send_inout(unic_dev->comdev.adev, &in, &out); + if (ret) + unic_err(unic_dev, + "failed to query vport ctx, offset = %u, ret = %d.\n", + offset, ret); + return ret; +} + int unic_set_fec_mode(struct unic_dev *unic_dev, u32 fec_mode) { struct unic_cfg_fec_cmd req = {0}; diff --git a/drivers/net/ub/unic/unic_hw.h b/drivers/net/ub/unic/unic_hw.h index ebe0c714549ddfa245e9e770eff821c78ca1658b..ba64d398e44b2ea7855d0f96b31a4edc4c82912a 100644 --- a/drivers/net/ub/unic/unic_hw.h +++ b/drivers/net/ub/unic/unic_hw.h @@ -11,6 +11,7 @@ #include #include "unic_dev.h" +#include "unic_qos_hw.h" #define UNIC_DL_CONFIG_MODE_TX_EN_B 0 #define UNIC_DL_CONFIG_MODE_RX_EN_B 1 @@ -94,6 +95,8 @@ int unic_set_promisc_mode(struct unic_dev *unic_dev, struct unic_promisc_en *promisc_en); int unic_cfg_vport_buf(struct unic_dev *unic_dev, bool init); +int unic_query_vport_ctx(struct unic_dev *unic_dev, u16 offset, + struct unic_vport_ctx_cmd *resp); int unic_set_fec_mode(struct unic_dev *unic_dev, u32 fec_mode); int unic_update_fec_stats(struct unic_dev *unic_dev); int unic_set_rss_tc_mode(struct unic_dev *unic_dev, u8 tc_vaild); diff --git a/drivers/net/ub/unic/unic_netdev.c b/drivers/net/ub/unic/unic_netdev.c index d8680ff9f894a9ca43757b727d7d4d0d1a314ebc..d2d213c90d0b10afc81e77ef2afbe3a23a8f96b3 100644 --- a/drivers/net/ub/unic/unic_netdev.c +++ b/drivers/net/ub/unic/unic_netdev.c @@ -164,6 +164,27 @@ static int unic_net_up(struct net_device *netdev) return 0; } +static void unic_link_status_record(struct net_device *netdev, bool linkup) +{ + struct unic_dev *unic_dev = netdev_priv(netdev); + struct unic_link_stats *record = &unic_dev->stats.link_record; + u64 idx, total; + + mutex_lock(&record->lock); + + if (linkup) + record->link_up_cnt++; + else + record->link_down_cnt++; + + total = record->link_up_cnt + record->link_down_cnt; + idx = (total - 1) % LINK_STAT_MAX_IDX; + record->stats[idx].link_tv_sec = ktime_get_real_seconds(); + record->stats[idx].link_status = linkup; + + mutex_unlock(&record->lock); +} + static void unic_clear_fec_stats(struct unic_dev *unic_dev) { struct unic_fec_stats *fec_stats = &unic_dev->stats.fec_stats; @@ -194,6 +215,8 @@ void unic_link_status_change(struct net_device *netdev, bool linkup) out: if (netif_msg_link(unic_dev)) unic_info(unic_dev, "%s.\n", linkup ? "link up" : "link down"); + + unic_link_status_record(netdev, linkup); } void unic_link_status_update(struct unic_dev *unic_dev) diff --git a/drivers/net/ub/unic/unic_qos_hw.c b/drivers/net/ub/unic/unic_qos_hw.c index 2f463be3d3242bdbd81ab81014352b30cb7d75fa..bba05964156b664c9751fe75f45894c8b7918172 100644 --- a/drivers/net/ub/unic/unic_qos_hw.c +++ b/drivers/net/ub/unic/unic_qos_hw.c @@ -29,6 +29,24 @@ int unic_set_hw_vl_map(struct unic_dev *unic_dev, u8 *dscp_vl, u8 *prio_vl, return ret; } +int unic_query_vl_map(struct unic_dev *unic_dev, + struct unic_config_vl_map_cmd *resp) +{ + struct unic_config_vl_map_cmd req = {0}; + struct ubase_cmd_buf in, out; + int ret; + + ubase_fill_inout_buf(&in, UBASE_OPC_CFG_VL_MAP, true, sizeof(req), + &req); + ubase_fill_inout_buf(&out, UBASE_OPC_CFG_VL_MAP, false, sizeof(*resp), + resp); + ret = ubase_cmd_send_inout(unic_dev->comdev.adev, &in, &out); + if (ret) + unic_err(unic_dev, "failed to query vl map, ret = %d.\n", ret); + + return ret; +} + /* vl_maxrate: byte per second */ int unic_config_vl_rate_limit(struct unic_dev *unic_dev, u64 *vl_maxrate, u16 vl_bitmap) @@ -41,7 +59,6 @@ int unic_config_vl_rate_limit(struct unic_dev *unic_dev, u64 *vl_maxrate, u32 vl_rate; int i, ret; - req.bus_ue_id = cpu_to_le16(USHRT_MAX); req.vl_bitmap = cpu_to_le16(vl_bitmap); for (i = 0; i < caps->vl_num; i++) { vl_rate = vl_maxrate[i] / UNIC_MBYTE_PER_SEND; @@ -54,7 +71,7 @@ int unic_config_vl_rate_limit(struct unic_dev *unic_dev, u64 *vl_maxrate, sizeof(req), &req); ret = ubase_cmd_send_in(adev, &in); - if (ret) + if (ret && ret != -EPERM) dev_err(adev->dev.parent, "failed to config vl rate limit, ret = %d.\n", ret); diff --git a/drivers/net/ub/unic/unic_qos_hw.h b/drivers/net/ub/unic/unic_qos_hw.h index 9f69474633493d200f75c6de7357f79b962afceb..82822cc871b60f8920e5d695478359839acf3eaf 100644 --- a/drivers/net/ub/unic/unic_qos_hw.h +++ b/drivers/net/ub/unic/unic_qos_hw.h @@ -13,6 +13,8 @@ int unic_set_hw_vl_map(struct unic_dev *unic_dev, u8 *dscp_vl, u8 *prio_vl, u8 map_type); +int unic_query_vl_map(struct unic_dev *unic_dev, + struct unic_config_vl_map_cmd *resp); int unic_config_vl_rate_limit(struct unic_dev *unic_dev, u64 *vl_maxrate, u16 vl_bitmap); diff --git a/drivers/net/ub/unic/unic_rack_ip.c b/drivers/net/ub/unic/unic_rack_ip.c index f2ff97304e78bc5e180b5daf83a779641431ed20..529856dcff7339ba753196bce585570a8e4b6004 100644 --- a/drivers/net/ub/unic/unic_rack_ip.c +++ b/drivers/net/ub/unic/unic_rack_ip.c @@ -41,7 +41,7 @@ static void unic_update_rack_addr_state(struct unic_vport *vport, addr_node->state = UNIC_COMM_ADDR_TO_ADD; unic_format_masked_ip_addr(format_masked_ip_addr, addr); unic_info(unic_dev, - "deleted an existing ip %s by accident and need to add it.\n", + "stack deleted an planned ip %s, need to re-add it.\n", format_masked_ip_addr); } break; @@ -90,7 +90,7 @@ static int unic_update_stack_ip_addr(struct unic_vport *vport, list_add_tail(&addr_node->node, list); unic_format_masked_ip_addr(format_masked_ip_addr, addr); unic_info(unic_dev, - "added a new ip %s by accident and need to delete it.\n", + "stack added a non-planned ip %s, need to delete it.\n", format_masked_ip_addr); set_bit(UNIC_VPORT_STATE_IP_TBL_CHANGE, &vport->state); goto unlock_and_exit; @@ -469,6 +469,9 @@ int unic_handle_notify_ip_event(struct auxiliary_device *adev, u8 service_ver, struct unic_stack_ip_info st_ip; int ret; + if (service_ver != UBASE_CTRLQ_SER_VER_01) + return -EOPNOTSUPP; + if (len < sizeof(*req)) { unic_err(priv, "failed to verify ip info size, len = %u.\n", len); return -EINVAL; diff --git a/drivers/net/ub/unic/unic_reset.c b/drivers/net/ub/unic/unic_reset.c new file mode 100644 index 0000000000000000000000000000000000000000..ff1239cc5131b160f2adadc65731bb99a3fff675 --- /dev/null +++ b/drivers/net/ub/unic/unic_reset.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + * + */ + +#include + +#include "unic_cmd.h" +#include "unic_dev.h" +#include "unic_hw.h" +#include "unic_netdev.h" +#include "unic_rack_ip.h" +#include "unic_reset.h" + +static void unic_dev_suspend(struct unic_dev *unic_dev) +{ + unic_uninit_channels(unic_dev); +} + +static void unic_reset_down(struct auxiliary_device *adev) +{ + struct unic_dev *priv = (struct unic_dev *)dev_get_drvdata(&adev->dev); + struct net_device *netdev = priv->comdev.netdev; + bool if_running; + int ret; + + if (!test_bit(UNIC_STATE_INITED, &priv->state) || + test_and_set_bit(UNIC_STATE_DISABLED, &priv->state)) { + unic_warn(priv, "failed to reset unic, device is not ready.\n"); + return; + } + + set_bit(UNIC_STATE_RESETTING, &priv->state); + if_running = netif_running(netdev); + + unic_info(priv, "unic reset start.\n"); + + unic_remove_period_task(priv); + + /* due to lack of cmdq when resetting, need to close promisc first, + * to prevent that concurrent deactivate event ubable to close promisc + * when resetting + */ + ret = unic_activate_promisc_mode(priv, false); + if (ret) + unic_warn(priv, "failed to close promisc, ret = %d.\n", ret); + else + set_bit(UNIC_VPORT_STATE_PROMISC_CHANGE, &priv->vport.state); + + rtnl_lock(); + ret = if_running ? unic_net_stop(netdev) : 0; + rtnl_unlock(); + if (ret) + unic_err(priv, "failed to stop unic net, ret = %d.\n", ret); +} + +static void unic_reset_uninit(struct auxiliary_device *adev) +{ + struct unic_dev *priv = (struct unic_dev *)dev_get_drvdata(&adev->dev); + + if (!test_bit(UNIC_STATE_RESETTING, &priv->state)) + return; + + unic_dev_suspend(priv); +} + +static int unic_dev_resume(struct unic_dev *unic_dev) +{ + int ret; + + ret = unic_init_channels(unic_dev, unic_dev->channels.num); + if (ret) + unic_err(unic_dev, "failed to init channels, ret = %d.\n", ret); + + return ret; +} + +static void unic_reset_init(struct auxiliary_device *adev) +{ + struct unic_dev *priv = (struct unic_dev *)dev_get_drvdata(&adev->dev); + struct net_device *netdev = priv->comdev.netdev; + bool if_running; + int ret; + + if (!test_bit(UNIC_STATE_RESETTING, &priv->state)) + return; + + ret = unic_dev_resume(priv); + if (ret) + goto err_unic_resume; + + unic_query_rack_ip(adev); + unic_start_period_task(netdev); + + if_running = netif_running(netdev); + clear_bit(UNIC_STATE_RESETTING, &priv->state); + clear_bit(UNIC_STATE_DISABLED, &priv->state); + rtnl_lock(); + ret = if_running ? unic_net_open(netdev) : 0; + rtnl_unlock(); + if (ret) + unic_err(priv, "failed to up net, ret = %d.\n", ret); + + unic_info(priv, "unic reset done.\n"); + + return; + +err_unic_resume: + clear_bit(UNIC_STATE_RESETTING, &priv->state); + clear_bit(UNIC_STATE_DISABLED, &priv->state); +} + +void unic_reset_handler(struct auxiliary_device *adev, + enum ubase_reset_stage stage) +{ + switch (stage) { + case UBASE_RESET_STAGE_DOWN: + unic_reset_down(adev); + break; + case UBASE_RESET_STAGE_UNINIT: + unic_reset_uninit(adev); + break; + case UBASE_RESET_STAGE_INIT: + unic_reset_init(adev); + break; + default: + break; + } +} diff --git a/drivers/net/ub/unic/unic_reset.h b/drivers/net/ub/unic/unic_reset.h new file mode 100644 index 0000000000000000000000000000000000000000..4ce313ac58aaa1fc5d6ab816ca01cad81e5163d9 --- /dev/null +++ b/drivers/net/ub/unic/unic_reset.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + * + */ + +#ifndef __UNIC_RESET_H__ +#define __UNIC_RESET_H__ + +#include + +void unic_reset_handler(struct auxiliary_device *adev, enum ubase_reset_stage stage); + +#endif diff --git a/drivers/net/ub/unic/unic_stats.c b/drivers/net/ub/unic/unic_stats.c index 8167bb6a6fcac9171211a70c9ef13797611d0683..4647ba5e3e5ed682ec71ccff92f0a7a47efc8fb0 100644 --- a/drivers/net/ub/unic/unic_stats.c +++ b/drivers/net/ub/unic/unic_stats.c @@ -14,6 +14,429 @@ #include "unic_netdev.h" #include "unic_stats.h" +static u32 cmdq_regs_addr[] = { + UNIC_TX_CMDQ_DEPTH, + UNIC_TX_CMDQ_TAIL, + UNIC_TX_CMDQ_HEAD, + UNIC_RX_CMDQ_DEPTH, + UNIC_RX_CMDQ_TAIL, + UNIC_RX_CMDQ_HEAD, + UNIC_CMDQ_INT_GEN, + UNIC_CMDQ_INT_SCR, + UNIC_CMDQ_INT_MASK, + UNIC_CMDQ_INT_STS, +}; + +static u32 ctrlq_regs_addr[] = { + UNIC_TX_CTRLQ_DEPTH, + UNIC_TX_CTRLQ_TAIL, + UNIC_TX_CTRLQ_HEAD, + UNIC_RX_CTRLQ_DEPTH, + UNIC_RX_CTRLQ_TAIL, + UNIC_RX_CTRLQ_HEAD, + UNIC_CTRLQ_INT_GEN, + UNIC_CTRLQ_INT_SCR, + UNIC_CTRLQ_INT_MASK, + UNIC_CTRLQ_INT_STS, +}; + +static const struct unic_res_regs_group unic_res_reg_arr[] = { + { + UNIC_TAG_CMDQ, cmdq_regs_addr, ARRAY_SIZE(cmdq_regs_addr), + NULL + }, { + UNIC_TAG_CTRLQ, ctrlq_regs_addr, ARRAY_SIZE(ctrlq_regs_addr), + ubase_adev_ctrlq_supported + }, +}; + +static bool unic_dfx_reg_support(struct unic_dev *unic_dev, u32 property) +{ + if (((property & UBASE_SUP_UBL) && unic_dev_ubl_supported(unic_dev)) || + ((property & UBASE_SUP_ETH) && unic_dev_eth_mac_supported(unic_dev))) + return true; + + return false; +} + +static struct unic_dfx_regs_group unic_dfx_reg_arr[] = { + { + UNIC_REG_NUM_IDX_TA, UNIC_TAG_TA, UBASE_OPC_DFX_TA_REG, + UBASE_SUP_UBL_ETH, unic_dfx_reg_support + }, { + UNIC_REG_NUM_IDX_TP, UNIC_TAG_TP, UBASE_OPC_DFX_TP_REG, + UBASE_SUP_UBL_ETH, unic_dfx_reg_support + }, { + UNIC_REG_NUM_IDX_BA, UNIC_TAG_BA, UBASE_OPC_DFX_BA_REG, + UBASE_SUP_UBL_ETH, unic_dfx_reg_support + }, { + UNIC_REG_NUM_IDX_NL, UNIC_TAG_NL, UBASE_OPC_DFX_NL_REG, + UBASE_SUP_UBL_ETH, unic_dfx_reg_support + }, { + UNIC_REG_NUM_IDX_DL, UNIC_TAG_DL, UBASE_OPC_DFX_DL_REG, + UBASE_SUP_UBL, unic_dfx_reg_support + }, +}; + +static const struct unic_stats_desc unic_sq_stats_str[] = { + {"pad_err", UNIC_SQ_STATS_FIELD_OFF(pad_err)}, + {"packets", UNIC_SQ_STATS_FIELD_OFF(packets)}, + {"bytes", UNIC_SQ_STATS_FIELD_OFF(bytes)}, + {"busy", UNIC_SQ_STATS_FIELD_OFF(busy)}, + {"more", UNIC_SQ_STATS_FIELD_OFF(more)}, + {"restart_queue", UNIC_SQ_STATS_FIELD_OFF(restart_queue)}, + {"over_max_sge_num", UNIC_SQ_STATS_FIELD_OFF(over_max_sge_num)}, + {"csum_err", UNIC_SQ_STATS_FIELD_OFF(csum_err)}, + {"ci_mismatch", UNIC_SQ_STATS_FIELD_OFF(ci_mismatch)}, + {"vlan_err", UNIC_SQ_STATS_FIELD_OFF(vlan_err)}, + {"fd_cnt", UNIC_SQ_STATS_FIELD_OFF(fd_cnt)}, + {"drop_cnt", UNIC_SQ_STATS_FIELD_OFF(drop_cnt)}, + {"cfg5_drop_cnt", UNIC_SQ_STATS_FIELD_OFF(cfg5_drop_cnt)} +}; + +static const struct unic_stats_desc unic_rq_stats_str[] = { + {"alloc_skb_err", UNIC_RQ_STATS_FIELD_OFF(alloc_skb_err)}, + {"packets", UNIC_RQ_STATS_FIELD_OFF(packets)}, + {"bytes", UNIC_RQ_STATS_FIELD_OFF(bytes)}, + {"err_pkt_len_cnt", UNIC_RQ_STATS_FIELD_OFF(err_pkt_len_cnt)}, + {"doi_cnt", UNIC_RQ_STATS_FIELD_OFF(doi_cnt)}, + {"trunc_cnt", UNIC_RQ_STATS_FIELD_OFF(trunc_cnt)}, + {"multicast", UNIC_RQ_STATS_FIELD_OFF(multicast)}, + {"l2_err", UNIC_RQ_STATS_FIELD_OFF(l2_err)}, + {"l3_l4_csum_err", UNIC_RQ_STATS_FIELD_OFF(l3_l4_csum_err)}, + {"alloc_frag_err", UNIC_RQ_STATS_FIELD_OFF(alloc_frag_err)}, + {"csum_complete", UNIC_RQ_STATS_FIELD_OFF(csum_complete)}, +}; + +static int unic_get_dfx_reg_num(struct unic_dev *unic_dev, u32 *reg_num, + u32 reg_arr_size) +{ + struct ubase_cmd_buf in, out; + int ret; + + ubase_fill_inout_buf(&in, UBASE_OPC_DFX_REG_NUM, true, 0, NULL); + ubase_fill_inout_buf(&out, UBASE_OPC_DFX_REG_NUM, true, + reg_arr_size * sizeof(u32), reg_num); + ret = ubase_cmd_send_inout(unic_dev->comdev.adev, &in, &out); + if (ret) + unic_err(unic_dev, + "failed to query dfx reg num, ret = %d.\n", ret); + + return ret; +} + +static int unic_get_res_regs_len(struct unic_dev *unic_dev, + const struct unic_res_regs_group *reg_arr, + u32 reg_arr_size) +{ + u32 i, count = 0; + + for (i = 0; i < reg_arr_size; i++) { + if (reg_arr[i].is_supported && + !reg_arr[i].is_supported(unic_dev->comdev.adev)) + continue; + + count += reg_arr[i].regs_count * sizeof(u32) + + sizeof(struct unic_tlv_hdr); + } + + return count; +} + +static int unic_get_dfx_regs_len(struct unic_dev *unic_dev, + struct unic_dfx_regs_group *reg_arr, + u32 reg_arr_size, u32 *reg_num) +{ + u32 i, count = 0; + + for (i = 0; i < reg_arr_size; i++) { + if (!reg_arr[i].is_supported(unic_dev, reg_arr[i].property)) + continue; + + count += sizeof(struct unic_tlv_hdr) + sizeof(u32) * + reg_num[reg_arr[i].regs_idx]; + } + + return count; +} + +int unic_get_regs_len(struct net_device *netdev) +{ + struct unic_dev *unic_dev = netdev_priv(netdev); + u32 reg_arr_size; + int count = 0; + u32 *reg_num; + int ret; + + if (unic_resetting(netdev)) + return -EBUSY; + + count += unic_get_res_regs_len(unic_dev, unic_res_reg_arr, + ARRAY_SIZE(unic_res_reg_arr)); + reg_arr_size = ARRAY_SIZE(unic_dfx_reg_arr); + reg_num = kcalloc(reg_arr_size, sizeof(u32), GFP_KERNEL); + if (!reg_num) + return -ENOMEM; + + ret = unic_get_dfx_reg_num(unic_dev, reg_num, reg_arr_size); + if (ret) { + kfree(reg_num); + return -EBUSY; + } + + count += unic_get_dfx_regs_len(unic_dev, unic_dfx_reg_arr, + reg_arr_size, reg_num); + kfree(reg_num); + + return count; +} + +static u16 unic_fetch_res_regs(struct unic_dev *unic_dev, u8 *data, u16 tag, + u32 *regs_addr_arr, u32 reg_num) +{ + struct unic_tlv_hdr *tlv = (struct unic_tlv_hdr *)data; + u32 *reg = (u32 *)(data + sizeof(struct unic_tlv_hdr)); + u32 i; + + tlv->tag = tag; + tlv->len = sizeof(struct unic_tlv_hdr) + reg_num * sizeof(u32); + + for (i = 0; i < reg_num; i++) + *reg++ = unic_read_reg(unic_dev, regs_addr_arr[i]); + + return tlv->len; +} + +static u32 unic_get_res_regs(struct unic_dev *unic_dev, u8 *data) +{ + u32 i, data_len = 0; + + for (i = 0; i < ARRAY_SIZE(unic_res_reg_arr); i++) { + if (unic_res_reg_arr[i].is_supported && + !unic_res_reg_arr[i].is_supported(unic_dev->comdev.adev)) + continue; + + data_len += unic_fetch_res_regs(unic_dev, data + data_len, + unic_res_reg_arr[i].tag, + unic_res_reg_arr[i].regs_addr, + unic_res_reg_arr[i].regs_count); + } + + return data_len; +} + +static int unic_query_regs_data(struct unic_dev *unic_dev, u8 *data, + u32 reg_num, u16 opcode) +{ + u32 *reg = (u32 *)(data + sizeof(struct unic_tlv_hdr)); + struct ubase_cmd_buf in, out; + u32 *out_regs; + int ret; + u32 i; + + out_regs = kcalloc(reg_num, sizeof(u32), GFP_KERNEL); + if (!out_regs) + return -ENOMEM; + + ubase_fill_inout_buf(&in, opcode, true, 0, NULL); + ubase_fill_inout_buf(&out, opcode, true, reg_num * sizeof(u32), + out_regs); + ret = ubase_cmd_send_inout(unic_dev->comdev.adev, &in, &out); + if (ret) { + unic_err(unic_dev, + "failed to send getting reg cmd(0x%x), ret = %d.\n", + opcode, ret); + goto err_send_cmd; + } + + for (i = 0; i < reg_num; i++) + *reg++ = le32_to_cpu(*(out_regs + i)); + +err_send_cmd: + kfree(out_regs); + + return ret; +} + +static int unic_get_dfx_regs(struct unic_dev *unic_dev, u8 *data, + struct unic_dfx_regs_group *reg_arr, + u32 reg_arr_size, u32 *reg_num) +{ + struct unic_tlv_hdr *tlv; + u16 idx; + int ret; + u32 i; + + for (i = 0; i < reg_arr_size; i++) { + if (!reg_arr[i].is_supported(unic_dev, reg_arr[i].property)) + continue; + + idx = reg_arr[i].regs_idx; + ret = unic_query_regs_data(unic_dev, data, reg_num[idx], + reg_arr[i].opcode); + if (ret) { + unic_err(unic_dev, + "failed to query dfx regs, ret = %d.\n", ret); + return ret; + } + + tlv = (struct unic_tlv_hdr *)data; + tlv->tag = reg_arr[i].tag; + tlv->len = sizeof(*tlv) + reg_num[idx] * sizeof(u32); + data += tlv->len; + } + + return 0; +} + +void unic_get_regs(struct net_device *netdev, struct ethtool_regs *cmd, + void *data) +{ + struct unic_dev *unic_dev = netdev_priv(netdev); + u8 *pdata = (u8 *)data; + u32 reg_arr_size; + u32 *reg_num; + int ret; + + if (unic_resetting(netdev)) { + unic_err(unic_dev, "dev resetting, could not get regs.\n"); + return; + } + + reg_arr_size = ARRAY_SIZE(unic_dfx_reg_arr); + reg_num = kcalloc(reg_arr_size, sizeof(u32), GFP_KERNEL); + if (!reg_num) { + unic_err(unic_dev, "failed to alloc reg num array.\n"); + return; + } + + pdata += unic_get_res_regs(unic_dev, pdata); + ret = unic_get_dfx_reg_num(unic_dev, reg_num, reg_arr_size); + if (ret) { + kfree(reg_num); + return; + } + + ret = unic_get_dfx_regs(unic_dev, pdata, unic_dfx_reg_arr, + reg_arr_size, reg_num); + if (ret) + unic_err(unic_dev, "failed to get dfx regs, ret = %d.\n", ret); + + kfree(reg_num); +} + +static u64 *unic_get_queues_stats(struct unic_dev *unic_dev, + const struct unic_stats_desc *stats, + u32 stats_size, enum unic_queue_type type, + u64 *data) +{ + struct unic_channel *c; + u32 i, j; + u8 *q; + + for (i = 0; i < unic_dev->channels.num; i++) { + c = &unic_dev->channels.c[i]; + q = (type == UNIC_QUEUE_TYPE_SQ) ? (u8 *)c->sq : (u8 *)c->rq; + for (j = 0; j < stats_size; j++) { + *data = UNIC_STATS_READ(q, stats[j].offset); + data++; + } + } + + return data; +} + +void unic_get_stats(struct net_device *netdev, + struct ethtool_stats *stats, u64 *data) +{ + struct unic_dev *unic_dev = netdev_priv(netdev); + u64 *p = data; + + if (unic_resetting(netdev) || !unic_dev->channels.c) { + unic_err(unic_dev, + "dev resetting or channel is null, could not get stats.\n"); + return; + } + + p = unic_get_queues_stats(unic_dev, unic_sq_stats_str, + ARRAY_SIZE(unic_sq_stats_str), + UNIC_QUEUE_TYPE_SQ, p); + + p = unic_get_queues_stats(unic_dev, unic_rq_stats_str, + ARRAY_SIZE(unic_rq_stats_str), + UNIC_QUEUE_TYPE_RQ, p); +} + +static u8 *unic_get_strings(u8 *data, const char *prefix, u32 num, + const struct unic_stats_desc *strs, u32 stats_size) +{ + u32 i, j; + + for (i = 0; i < num; i++) { + for (j = 0; j < stats_size; j++) { + data[ETH_GSTRING_LEN - 1] = '\0'; + + if (prefix) + scnprintf(data, ETH_GSTRING_LEN - 1, "%s%u_%s", + prefix, i, strs[j].desc); + else + scnprintf(data, ETH_GSTRING_LEN - 1, "%s", + strs[j].desc); + + data += ETH_GSTRING_LEN; + } + } + + return data; +} + +static u8 *unic_get_queues_strings(struct unic_dev *unic_dev, u8 *data) +{ + u32 channel_num = unic_dev->channels.num; + + /* get desc for Tx */ + data = unic_get_strings(data, "txq", channel_num, unic_sq_stats_str, + ARRAY_SIZE(unic_sq_stats_str)); + + /* get desc for Rx */ + data = unic_get_strings(data, "rxq", channel_num, unic_rq_stats_str, + ARRAY_SIZE(unic_rq_stats_str)); + + return data; +} + +void unic_get_stats_strings(struct net_device *netdev, u32 stringset, u8 *data) +{ + struct unic_dev *unic_dev = netdev_priv(netdev); + u8 *p = data; + + switch (stringset) { + case ETH_SS_STATS: + p = unic_get_queues_strings(unic_dev, p); + break; + default: + break; + } +} + +int unic_get_sset_count(struct net_device *netdev, int stringset) +{ + struct unic_dev *unic_dev = netdev_priv(netdev); + u32 channel_num = unic_dev->channels.num; + int count; + + switch (stringset) { + case ETH_SS_STATS: + count = ARRAY_SIZE(unic_sq_stats_str) * channel_num; + count += ARRAY_SIZE(unic_rq_stats_str) * channel_num; + break; + default: + return -EOPNOTSUPP; + } + + return count; +} + static void unic_get_fec_stats_total(struct unic_dev *unic_dev, u8 stats_flags, struct ethtool_fec_stats *fec_stats) { diff --git a/drivers/net/ub/unic/unic_stats.h b/drivers/net/ub/unic/unic_stats.h index d9d9f0b0ee3bff98e99906161307680d3f9699cd..623b732f3d8e067cc7529731316002a78c43e1ff 100644 --- a/drivers/net/ub/unic/unic_stats.h +++ b/drivers/net/ub/unic/unic_stats.h @@ -12,10 +12,101 @@ #include #include +#define UNIC_SQ_STATS_FIELD_OFF(fld) (offsetof(struct unic_sq, stats) + \ + offsetof(struct unic_sq_stats, fld)) +#define UNIC_RQ_STATS_FIELD_OFF(fld) (offsetof(struct unic_rq, stats) + \ + offsetof(struct unic_rq_stats, fld)) + +#define UNIC_STATS_READ(p, offset) (*(u64 *)((u8 *)(p) + (offset))) + #define UNIC_FEC_CORR_BLOCKS BIT(0) #define UNIC_FEC_UNCORR_BLOCKS BIT(1) #define UNIC_FEC_CORR_BITS BIT(2) +#define UNIC_TX_CMDQ_DEPTH UBASE_CSQ_DEPTH_REG +#define UNIC_TX_CMDQ_TAIL UBASE_CSQ_TAIL_REG +#define UNIC_TX_CMDQ_HEAD UBASE_CSQ_HEAD_REG +#define UNIC_RX_CMDQ_DEPTH UBASE_CRQ_DEPTH_REG +#define UNIC_RX_CMDQ_TAIL UBASE_CRQ_TAIL_REG +#define UNIC_RX_CMDQ_HEAD UBASE_CRQ_HEAD_REG +#define UNIC_CMDQ_INT_GEN 0x18000 +#define UNIC_CMDQ_INT_SCR 0x18004 +#define UNIC_CMDQ_INT_MASK 0x18008 +#define UNIC_CMDQ_INT_STS 0x1800c + +#define UNIC_TX_CTRLQ_DEPTH UBASE_CTRLQ_CSQ_DEPTH_REG +#define UNIC_TX_CTRLQ_TAIL UBASE_CTRLQ_CSQ_TAIL_REG +#define UNIC_TX_CTRLQ_HEAD UBASE_CTRLQ_CSQ_HEAD_REG +#define UNIC_RX_CTRLQ_DEPTH UBASE_CTRLQ_CRQ_DEPTH_REG +#define UNIC_RX_CTRLQ_TAIL UBASE_CTRLQ_CRQ_TAIL_REG +#define UNIC_RX_CTRLQ_HEAD UBASE_CTRLQ_CRQ_HEAD_REG +#define UNIC_CTRLQ_INT_GEN 0x18010 +#define UNIC_CTRLQ_INT_SCR 0x18014 +#define UNIC_CTRLQ_INT_MASK 0x18018 +#define UNIC_CTRLQ_INT_STS 0x1801c + +enum unic_reg_num_idx { + UNIC_REG_NUM_IDX_DL = 0, + UNIC_REG_NUM_IDX_NL, + UNIC_REG_NUM_IDX_BA, + UNIC_REG_NUM_IDX_TP, + UNIC_REG_NUM_IDX_TA, + UNIC_REG_NUM_IDX_MAX, +}; + +enum unic_reg_tag { + UNIC_TAG_CMDQ = 0, + UNIC_TAG_CTRLQ, + UNIC_TAG_DL, + UNIC_TAG_NL, + UNIC_TAG_BA, + UNIC_TAG_TP, + UNIC_TAG_TA, + UNIC_TAG_MAX, +}; + +enum unic_queue_type { + UNIC_QUEUE_TYPE_SQ = 0, + UNIC_QUEUE_TYPE_RQ, +}; + +struct unic_res_regs_group { + u16 tag; + u32 *regs_addr; + u32 regs_count; + bool (*is_supported)(struct auxiliary_device *adev); +}; + +struct unic_dump_reg_hdr { + u8 flag; + u8 rsv[3]; +}; + +struct unic_tlv_hdr { + u16 tag; + u16 len; +}; + +struct unic_dfx_regs_group { + u16 regs_idx; + u16 tag; + u16 opcode; + u32 property; + bool (*is_supported)(struct unic_dev *unic_dev, u32 property); +}; + +struct unic_stats_desc { + char desc[ETH_GSTRING_LEN]; + u16 offset; +}; + +int unic_get_regs_len(struct net_device *netdev); +void unic_get_regs(struct net_device *netdev, struct ethtool_regs *cmd, + void *data); +void unic_get_stats_strings(struct net_device *netdev, u32 stringset, u8 *data); +int unic_get_sset_count(struct net_device *netdev, int stringset); +void unic_get_stats(struct net_device *netdev, + struct ethtool_stats *stats, u64 *data); void unic_get_fec_stats(struct net_device *ndev, struct ethtool_fec_stats *fec_stats); diff --git a/drivers/ub/Kconfig b/drivers/ub/Kconfig index 0b7cb0ef16cf335b25930602f3425c0cb428e8cd..5aaa3bcc014a74a74ebf27429d9b418af9fb75d2 100644 --- a/drivers/ub/Kconfig +++ b/drivers/ub/Kconfig @@ -16,8 +16,10 @@ if UB source "drivers/ub/ubus/Kconfig" source "drivers/ub/ubfi/Kconfig" source "drivers/ub/ubase/Kconfig" +source "drivers/ub/cdma/Kconfig" source "drivers/ub/obmm/Kconfig" source "drivers/ub/sentry/Kconfig" +source "drivers/ub/urma/hw/udma/Kconfig" config UB_URMA tristate "Unified Bus (UB) urma support" default m diff --git a/drivers/ub/Makefile b/drivers/ub/Makefile index d1dd2267abe0f2eaf2f9d4952ddd4a5876cb49f3..1725f006d1974bb6bc45ba9acdaac09d575a3cb0 100644 --- a/drivers/ub/Makefile +++ b/drivers/ub/Makefile @@ -4,5 +4,7 @@ obj-y += ubus/ obj-y += ubfi/ obj-$(CONFIG_UB_URMA) += urma/ obj-$(CONFIG_UB_UBASE) += ubase/ +obj-$(CONFIG_UB_CDMA) += cdma/ +obj-$(CONFIG_UB_UDMA) += urma/hw/udma/ obj-y += obmm/ obj-$(CONFIG_UB_SENTRY) += sentry/ diff --git a/drivers/ub/cdma/Kconfig b/drivers/ub/cdma/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..b09e132158525aba9ee46eb5f9e1a2cb487eab35 --- /dev/null +++ b/drivers/ub/cdma/Kconfig @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0+ +menuconfig UB_CDMA + default n + tristate "cdma driver" + depends on UB_UBASE && UB_UMMU_CORE + help + This option enables support for CDMA drivers. The CDMA driver facilitates + the creation and destruction of CDMA devices, as well as the creation of + resources within CDMA devices to perform DMA read/write tasks and retrieve + the completion status of executed tasks. diff --git a/drivers/ub/cdma/Makefile b/drivers/ub/cdma/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..88dc9946a09250c5a0e7b84b2cff171d115d16c0 --- /dev/null +++ b/drivers/ub/cdma/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0+ + +cdma-$(CONFIG_UB_CDMA) := cdma_main.o cdma_dev.o cdma_chardev.o cdma_cmd.o cdma_tid.o cdma_ioctl.o \ + cdma_api.o cdma_context.o cdma_queue.o cdma_uobj.o cdma_jfc.o cdma_common.o \ + cdma_db.o cdma_mbox.o cdma_tp.o cdma_jfs.o cdma_eq.o cdma_event.o cdma_segment.o \ + cdma_handle.o cdma_debugfs.o cdma_mmap.o + +obj-m += cdma.o diff --git a/drivers/ub/cdma/cdma.h b/drivers/ub/cdma/cdma.h new file mode 100644 index 0000000000000000000000000000000000000000..b7d00bcf39ac73d77beb80ba6b17d457376b2622 --- /dev/null +++ b/drivers/ub/cdma/cdma.h @@ -0,0 +1,233 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_H__ +#define __CDMA_H__ + +#include +#include +#include +#include +#include +#include "cdma_debugfs.h" +#include + +extern u32 jfc_arm_mode; +extern bool cqe_mode; +extern struct list_head g_client_list; +extern struct rw_semaphore g_clients_rwsem; +extern struct rw_semaphore g_device_rwsem; +extern struct mutex g_cdma_reset_mutex; + +#define CDMA_HW_PAGE_SHIFT 12 +#define CDMA_HW_PAGE_SIZE (1 << CDMA_HW_PAGE_SHIFT) + +#define CDMA_DEFAULT_CQE_SIZE 128 +#define CDMA_RESET_WAIT_TIME 3000 +#define CDMA_MAX_SL_NUM 16 + +#define CDMA_UPI_MASK 0x7FFF + +#define DMA_MAX_DEV_NAME 64 + +enum cdma_cqe_size { + CDMA_64_CQE_SIZE, + CDMA_128_CQE_SIZE, +}; + +enum cdma_status { + CDMA_NORMAL, + CDMA_SUSPEND, +}; + +enum cdma_client_ops { + CDMA_CLIENT_STOP, + CDMA_CLIENT_REMOVE, + CDMA_CLIENT_ADD, +}; + +enum { + CDMA_CAP_FEATURE_AR = BIT(0), + CDMA_CAP_FEATURE_JFC_INLINE = BIT(4), + CDMA_CAP_FEATURE_DIRECT_WQE = BIT(11), + CDMA_CAP_FEATURE_CONG_CTRL = BIT(16), +}; + +struct cdma_res { + u32 max_cnt; + u32 start_idx; + u32 depth; +}; + +struct cdma_oor_caps { + bool oor_en; + bool reorder_queue_en; + u8 reorder_cap; + u8 reorder_queue_shift; + u8 at_times; + u16 on_flight_size; +}; + +struct cdma_tbl { + u32 max_cnt; + u32 size; +}; + +struct cdma_caps { + struct cdma_res jfs; + struct cdma_res jfce; + struct cdma_res jfc; + struct cdma_res queue; + u32 jfs_sge; + u32 jfr_sge; + u32 jfs_rsge; + u32 jfs_inline_sz; + u32 comp_vector_cnt; + u32 eid_num; + u16 ue_cnt; + u8 ue_id; + u32 rc_outstd_cnt; + u32 utp_cnt; + u32 trans_mode; + u32 ta_version; + u32 tp_version; + u32 max_msg_len; + u32 feature; + u32 public_jetty_cnt; + u32 rsvd_jetty_cnt; + u16 cons_ctrl_alg; + u16 rc_queue_num; + u16 rc_queue_depth; + u8 rc_entry_size; + u8 packet_pattern_mode; + u8 ack_queue_num; + u8 port_num; + u8 cqe_size; + u8 cc_priority_cnt; + bool virtualization; + struct cdma_oor_caps oor_caps; + struct cdma_tbl src_addr; + struct cdma_tbl seid; +}; + +struct cdma_idr { + struct idr idr; + u32 min; + u32 max; + u32 next; +}; + +struct cdma_table { + spinlock_t lock; + struct cdma_idr idr_tbl; +}; + +struct cdma_chardev { + struct device *dev; + +#define CDMA_NAME_LEN 16 + char name[CDMA_NAME_LEN]; + struct cdev cdev; + int dev_num; + dev_t devno; +}; + +union cdma_umem_flag { + struct { + u32 non_pin : 1; /* 0: pinned to physical memory. 1: non pin. */ + u32 writable : 1; /* 0: read-only. 1: writable. */ + u32 reserved : 30; + } bs; + u32 value; +}; + +struct cdma_umem { + struct mm_struct *owning_mm; + union cdma_umem_flag flag; + struct sg_table sg_head; + struct cdma_dev *dev; + + u64 length; + u32 nmap; + u64 va; +}; + +struct cdma_buf { + dma_addr_t addr; /* pass to hw */ + union { + void *kva; /* used for kernel mode */ + struct iova_slot *slot; + void *kva_or_slot; + }; + void *aligned_va; + struct cdma_umem *umem; + u32 entry_cnt_mask; + u32 entry_cnt_mask_ilog2; + u32 entry_size; + u32 entry_cnt; + u32 cnt_per_page_shift; + struct xarray id_table_xa; + struct mutex id_table_mutex; +}; + +struct cdma_dev { + struct dma_device base; + struct device *dev; + struct auxiliary_device *adev; + struct cdma_chardev chardev; + struct cdma_caps caps; + struct cdma_dbgfs cdbgfs; + + u32 eid; + u32 upi; + u32 tid; + u32 ummu_tid; + u32 status; + u8 sl_num; + u8 sl[CDMA_MAX_SL_NUM]; + + void __iomem *k_db_base; + resource_size_t db_base; + struct iommu_sva *ksva; + /* ctx manager */ + struct list_head ctx_list; + struct idr ctx_idr; + spinlock_t ctx_lock; + struct mutex eu_mutex; + struct mutex db_mutex; + struct list_head db_page; + + struct cdma_table queue_table; + struct cdma_table ctp_table; + struct cdma_table jfs_table; + struct cdma_table jfc_table; + struct cdma_table jfce_table; + struct cdma_table seg_table; + struct ubase_event_nb *ae_event_addr[UBASE_EVENT_TYPE_MAX]; + struct mutex file_mutex; + struct list_head file_list; + struct page *arm_db_page; + atomic_t cmdcnt; + struct completion cmddone; +}; + +struct cdma_jfs_event { + struct list_head async_event_list; + u32 async_events_reported; +}; + +struct cdma_jfc_event { + struct cdma_base_jfc *jfc; + struct cdma_jfce *jfce; + struct list_head comp_event_list; + struct list_head async_event_list; + u32 comp_events_reported; + u32 async_events_reported; +}; + +static inline struct cdma_dev *get_cdma_dev(struct auxiliary_device *adev) +{ + return (struct cdma_dev *)dev_get_drvdata(&adev->dev); +} + +#endif /* _CDMA_H_ */ diff --git a/drivers/ub/cdma/cdma_api.c b/drivers/ub/cdma/cdma_api.c new file mode 100644 index 0000000000000000000000000000000000000000..ae84210c1f978d73df45b5d9dbcf2e9ebc0f0014 --- /dev/null +++ b/drivers/ub/cdma/cdma_api.c @@ -0,0 +1,935 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define pr_fmt(fmt) "CDMA: " fmt +#define dev_fmt pr_fmt + +#include +#include +#include "cdma_segment.h" +#include "cdma_dev.h" +#include "cdma_cmd.h" +#include "cdma_context.h" +#include "cdma_queue.h" +#include "cdma_jfc.h" +#include "cdma.h" +#include "cdma_handle.h" +#include + +LIST_HEAD(g_client_list); +DECLARE_RWSEM(g_clients_rwsem); +DECLARE_RWSEM(g_device_rwsem); + +/** + * dma_get_device_list - Get DMA device list + * @num_devices: DMA device number + * + * Users can perform subsequent resource creation operations using a pointer + * to a DMA device in the list. + * + * Context: Process context. + * Return: address of the first device in the list + */ +struct dma_device *dma_get_device_list(u32 *num_devices) +{ + struct cdma_device_attr *attr; + struct xarray *cdma_devs_tbl; + struct cdma_dev *cdev = NULL; + struct dma_device *ret_list; + unsigned long index; + u32 count = 0; + u32 devs_num; + + if (!num_devices) + return NULL; + + cdma_devs_tbl = get_cdma_dev_tbl(&devs_num); + if (!devs_num) { + pr_err("cdma device table is empty.\n"); + return NULL; + } + + ret_list = kcalloc(devs_num, sizeof(struct dma_device), GFP_KERNEL); + if (!ret_list) { + *num_devices = 0; + return NULL; + } + + xa_for_each(cdma_devs_tbl, index, cdev) { + attr = &cdev->base.attr; + if (cdev->status == CDMA_SUSPEND) { + pr_warn("cdma device is not prepared, eid = 0x%x.\n", + attr->eid.dw0); + continue; + } + + if (!attr->eu_num) { + pr_warn("no eu in cdma dev eid = 0x%x.\n", cdev->eid); + continue; + } + + memcpy(&attr->eu, &attr->eus[0], sizeof(attr->eu)); + attr->eid.dw0 = cdev->eid; + memcpy(&ret_list[count], &cdev->base, sizeof(*ret_list)); + ret_list[count].private_data = kzalloc( + sizeof(struct cdma_ctx_res), GFP_KERNEL); + if (!ret_list[count].private_data) + break; + count++; + } + *num_devices = count; + + return ret_list; +} +EXPORT_SYMBOL_GPL(dma_get_device_list); + +/** + * dma_free_device_list - Free DMA device list + * @dev_list: DMA device list + * @num_devices: DMA device number + * + * It can be called after using dev_list and must be called. + * + * Context: Process context. + * Return: NA + */ +void dma_free_device_list(struct dma_device *dev_list, u32 num_devices) +{ + int ref_cnt; + u32 i; + + if (!dev_list) + return; + + for (i = 0; i < num_devices; i++) { + ref_cnt = atomic_read(&dev_list[i].ref_cnt); + if (ref_cnt > 0) { + pr_warn("the device resourse is still in use, eid = 0x%x, cnt = %d.\n", + dev_list[i].attr.eid.dw0, ref_cnt); + return; + } + } + + for (i = 0; i < num_devices; i++) + kfree(dev_list[i].private_data); + + kfree(dev_list); +} +EXPORT_SYMBOL_GPL(dma_free_device_list); + +/** + * dma_get_device_by_eid - Get the specified EID DMA device + * @eid: Device eid pointer + * + * Choose one to use with the dma_get_device_list function. + * + * Context: Process context. + * Return: DMA device structure pointer + */ +struct dma_device *dma_get_device_by_eid(struct dev_eid *eid) +{ + struct cdma_device_attr *attr; + struct xarray *cdma_devs_tbl; + struct cdma_dev *cdev = NULL; + struct dma_device *ret_dev; + unsigned long index; + u32 devs_num; + + if (!eid) + return NULL; + + cdma_devs_tbl = get_cdma_dev_tbl(&devs_num); + if (!devs_num) { + pr_err("cdma device table is empty.\n"); + return NULL; + } + + ret_dev = kzalloc(sizeof(struct dma_device), GFP_KERNEL); + if (!ret_dev) + return NULL; + + xa_for_each(cdma_devs_tbl, index, cdev) { + attr = &cdev->base.attr; + if (cdev->status == CDMA_SUSPEND) { + pr_warn("cdma device is not prepared, eid = 0x%x.\n", + attr->eid.dw0); + continue; + } + + if (!cdma_find_seid_in_eus(attr->eus, attr->eu_num, eid, + &attr->eu)) + continue; + + memcpy(ret_dev, &cdev->base, sizeof(*ret_dev)); + ret_dev->private_data = kzalloc( + sizeof(struct cdma_ctx_res), GFP_KERNEL); + if (!ret_dev->private_data) { + kfree(ret_dev); + return NULL; + } + return ret_dev; + } + kfree(ret_dev); + + return NULL; +} +EXPORT_SYMBOL_GPL(dma_get_device_by_eid); + +/** + * dma_create_context - Create DMA context + * @dma_dev: DMA device pointer + * + * The context is used to store resources such as Queue and Segment, and + * returns a pointer to the context information. + * + * Context: Process context. + * Return: DMA context ID value + */ +int dma_create_context(struct dma_device *dma_dev) +{ + struct cdma_ctx_res *ctx_res; + struct cdma_context *ctx; + struct cdma_dev *cdev; + + if (!dma_dev || !dma_dev->private_data) { + pr_err("the dma_dev does not exist.\n"); + return -EINVAL; + } + + cdev = get_cdma_dev_by_eid(dma_dev->attr.eid.dw0); + if (!cdev) { + pr_err("can't find cdev by eid, eid = 0x%x\n", + dma_dev->attr.eid.dw0); + return -EINVAL; + } + + if (cdev->status == CDMA_SUSPEND) { + pr_warn("cdma device is not prepared, eid = 0x%x.\n", + dma_dev->attr.eid.dw0); + return -EINVAL; + } + + ctx_res = (struct cdma_ctx_res *)dma_dev->private_data; + if (ctx_res->ctx) { + pr_err("ctx has been created.\n"); + return -EEXIST; + } + + atomic_inc(&dma_dev->ref_cnt); + ctx = cdma_alloc_context(cdev, true); + if (IS_ERR(ctx)) { + pr_err("alloc context failed, ret = %ld\n", PTR_ERR(ctx)); + atomic_dec(&dma_dev->ref_cnt); + return PTR_ERR(ctx); + } + ctx_res->ctx = ctx; + + return ctx->handle; +} +EXPORT_SYMBOL_GPL(dma_create_context); + +/** + * dma_delete_context - Delete DMA context + * @dma_dev: DMA device pointe + * @handle: DMA context ID value + * Context: Process context. + * Return: NA + */ +void dma_delete_context(struct dma_device *dma_dev, int handle) +{ + struct cdma_ctx_res *ctx_res; + struct cdma_context *ctx; + struct cdma_dev *cdev; + int ref_cnt; + + if (!dma_dev || !dma_dev->private_data) + return; + + cdev = get_cdma_dev_by_eid(dma_dev->attr.eid.dw0); + if (!cdev) { + pr_err("can't find cdev by eid, eid = 0x%x\n", + dma_dev->attr.eid.dw0); + return; + } + + ctx_res = (struct cdma_ctx_res *)dma_dev->private_data; + ctx = ctx_res->ctx; + if (!ctx) { + dev_err(cdev->dev, "no context needed to be free\n"); + return; + } + + ref_cnt = atomic_read(&ctx->ref_cnt); + if (ref_cnt > 0) { + dev_warn(cdev->dev, + "context resourse is still in use, cnt = %d.\n", + ref_cnt); + return; + } + + cdma_free_context(cdev, ctx); + ctx_res->ctx = NULL; + atomic_dec(&dma_dev->ref_cnt); +} +EXPORT_SYMBOL_GPL(dma_delete_context); + +/** + * dma_alloc_queue - Alloc DMA queue + * @dma_dev: DMA device pointer + * @ctx_id: DMA context ID + * @cfg: DMA queue configuration information pointer + * + * The user uses the queue for DMA read and write operations. + * + * Context: Process context. + * Return: DMA queue ID value + */ +int dma_alloc_queue(struct dma_device *dma_dev, int ctx_id, struct queue_cfg *cfg) +{ + struct cdma_ctx_res *ctx_res; + struct cdma_queue *queue; + struct cdma_context *ctx; + struct cdma_dev *cdev; + int ret; + + if (!cfg || !dma_dev || !dma_dev->private_data) + return -EINVAL; + + cdev = get_cdma_dev_by_eid(dma_dev->attr.eid.dw0); + if (!cdev) { + pr_err("can't find cdev by eid, eid = 0x%x.\n", + dma_dev->attr.eid.dw0); + return -EINVAL; + } + + if (cdev->status == CDMA_SUSPEND) { + pr_warn("cdma device is not prepared, eid = 0x%x.\n", + dma_dev->attr.eid.dw0); + return -EINVAL; + } + + ctx = cdma_find_ctx_by_handle(cdev, ctx_id); + if (!ctx) { + dev_err(cdev->dev, "invalid ctx_id = %d.\n", ctx_id); + return -EINVAL; + } + atomic_inc(&ctx->ref_cnt); + + queue = cdma_create_queue(cdev, ctx, cfg, dma_dev->attr.eu.eid_idx, + true); + if (!queue) { + dev_err(cdev->dev, "create queue failed.\n"); + ret = -EINVAL; + goto decrease_cnt; + } + + ctx_res = (struct cdma_ctx_res *)dma_dev->private_data; + ret = xa_err( + xa_store(&ctx_res->queue_xa, queue->id, queue, GFP_KERNEL)); + if (ret) { + dev_err(cdev->dev, "store queue to ctx_res failed, ret = %d\n", + ret); + goto free_queue; + } + + return queue->id; + +free_queue: + cdma_delete_queue(cdev, queue->id); +decrease_cnt: + atomic_dec(&ctx->ref_cnt); + return ret; +} +EXPORT_SYMBOL_GPL(dma_alloc_queue); + +/** + * dma_free_queue - Free DMA queue + * @dma_dev: DMA device pointer + * @queue_id: DMA queue ID + * Context: Process context. + * Return: NA + */ +void dma_free_queue(struct dma_device *dma_dev, int queue_id) +{ + struct cdma_ctx_res *ctx_res; + struct cdma_context *ctx; + struct cdma_queue *queue; + struct cdma_dev *cdev; + + if (!dma_dev || !dma_dev->private_data) + return; + + cdev = get_cdma_dev_by_eid(dma_dev->attr.eid.dw0); + if (!cdev) { + pr_err("can't find cdev by eid, eid = 0x%x\n", + dma_dev->attr.eid.dw0); + return; + } + + ctx_res = (struct cdma_ctx_res *)dma_dev->private_data; + queue = (struct cdma_queue *)xa_load(&ctx_res->queue_xa, queue_id); + if (!queue) { + dev_err(cdev->dev, "no queue found in this device, id = %d\n", + queue_id); + return; + } + xa_erase(&ctx_res->queue_xa, queue_id); + ctx = queue->ctx; + + cdma_delete_queue(cdev, queue_id); + + atomic_dec(&ctx->ref_cnt); +} +EXPORT_SYMBOL_GPL(dma_free_queue); + +/** + * dma_register_seg - Register local segment + * @dma_dev: DMA device pointer + * @ctx_id: DMA context ID + * @cfg: DMA segment configuration information pointer + * + * The segment stores local payload information for operations such as DMA + * read and write, and returns a pointer to the segment information. + * + * Context: Process context. + * Return: DMA segment structure pointer + */ +struct dma_seg *dma_register_seg(struct dma_device *dma_dev, int ctx_id, + struct dma_seg_cfg *cfg) +{ + struct cdma_ctx_res *ctx_res; + struct cdma_segment *seg; + struct cdma_context *ctx; + struct dma_seg *ret_seg; + struct cdma_dev *cdev; + int ret; + + if (!dma_dev || !dma_dev->private_data || !cfg || !cfg->sva || !cfg->len) + return NULL; + + cdev = get_cdma_dev_by_eid(dma_dev->attr.eid.dw0); + if (!cdev) { + pr_err("can not find normal cdev by eid, eid = 0x%x\n", + dma_dev->attr.eid.dw0); + return NULL; + } + + if (cdev->status == CDMA_SUSPEND) { + pr_warn("cdma device is not prepared, eid = 0x%x.\n", + dma_dev->attr.eid.dw0); + return NULL; + } + + ctx = cdma_find_ctx_by_handle(cdev, ctx_id); + if (!ctx) { + dev_err(cdev->dev, "find ctx by handle failed, handle = %d.\n", + ctx_id); + return NULL; + } + atomic_inc(&ctx->ref_cnt); + + seg = cdma_register_seg(cdev, cfg, true); + if (!seg) + goto decrease_cnt; + + seg->ctx = ctx; + ret = cdma_seg_grant(cdev, seg, cfg); + if (ret) + goto unregister_seg; + + ret_seg = kzalloc(sizeof(struct dma_seg), GFP_KERNEL); + if (!ret_seg) + goto ungrant_seg; + + memcpy(ret_seg, &seg->base, sizeof(struct dma_seg)); + + ctx_res = (struct cdma_ctx_res *)dma_dev->private_data; + ret = xa_err(xa_store(&ctx_res->seg_xa, ret_seg->handle, seg, + GFP_KERNEL)); + if (ret) { + dev_err(cdev->dev, "store seg to ctx_res failed, ret = %d\n", + ret); + goto free_seg; + } + + return ret_seg; + +free_seg: + kfree(ret_seg); +ungrant_seg: + cdma_seg_ungrant(seg); +unregister_seg: + cdma_unregister_seg(cdev, seg); +decrease_cnt: + atomic_dec(&ctx->ref_cnt); + return NULL; +} +EXPORT_SYMBOL_GPL(dma_register_seg); + +/** + * dma_unregister_seg - Unregister local segment + * @dma_dev: DMA device pointer + * @dma_seg: DMA segment pointer + * Context: Process context. + * Return: NA + */ +void dma_unregister_seg(struct dma_device *dma_dev, struct dma_seg *dma_seg) +{ + struct cdma_ctx_res *ctx_res; + struct cdma_context *ctx; + struct cdma_segment *seg; + struct cdma_dev *cdev; + + if (!dma_dev || !dma_dev->private_data || !dma_seg) + return; + + cdev = get_cdma_dev_by_eid(dma_dev->attr.eid.dw0); + if (!cdev) { + pr_err("can not find cdev by eid, eid = 0x%x\n", + dma_dev->attr.eid.dw0); + return; + } + + ctx_res = (struct cdma_ctx_res *)dma_dev->private_data; + seg = xa_load(&ctx_res->seg_xa, dma_seg->handle); + if (!seg) { + dev_err(cdev->dev, + "no segment found in this device, handle = %llu\n", + dma_seg->handle); + return; + } + xa_erase(&ctx_res->seg_xa, dma_seg->handle); + ctx = seg->ctx; + + cdma_seg_ungrant(seg); + cdma_unregister_seg(cdev, seg); + kfree(dma_seg); + + atomic_dec(&ctx->ref_cnt); +} +EXPORT_SYMBOL_GPL(dma_unregister_seg); + +/** + * dma_import_seg - Import the remote segment + * @cfg: DMA segment configuration information pointer + * + * The segment stores the remote payload information for operations such as + * DMA read and write, and returns the segment information pointer. + * + * Context: Process context. + * Return: DMA segment structure pointer + */ +struct dma_seg *dma_import_seg(struct dma_seg_cfg *cfg) +{ + if (!cfg || !cfg->sva || !cfg->len) + return NULL; + + return cdma_import_seg(cfg); +} +EXPORT_SYMBOL_GPL(dma_import_seg); + +/** + * dma_unimport_seg - Unimport the remote segment + * @dma_seg: DMA segment pointer + * Context: Process context. + * Return: NA + */ +void dma_unimport_seg(struct dma_seg *dma_seg) +{ + if (!dma_seg) + return; + + cdma_unimport_seg(dma_seg); +} +EXPORT_SYMBOL_GPL(dma_unimport_seg); + +static int cdma_param_transfer(struct dma_device *dma_dev, int queue_id, + struct cdma_dev **cdev, + struct cdma_queue **cdma_queue) +{ + struct cdma_queue *tmp_q; + struct cdma_dev *tmp_dev; + u32 eid; + + eid = dma_dev->attr.eid.dw0; + tmp_dev = get_cdma_dev_by_eid(eid); + if (!tmp_dev) { + pr_err("get cdma dev failed, eid = 0x%x.\n", eid); + return -EINVAL; + } + + if (tmp_dev->status == CDMA_SUSPEND) { + pr_warn("cdma device is not prepared, eid = 0x%x.\n", eid); + return -EINVAL; + } + + tmp_q = cdma_find_queue(tmp_dev, queue_id); + if (!tmp_q) { + dev_err(tmp_dev->dev, "get resource failed.\n"); + return -EINVAL; + } + + if (!tmp_q->tp || !tmp_q->jfs || !tmp_q->jfc) { + dev_err(tmp_dev->dev, "get jetty parameters failed.\n"); + return -EFAULT; + } + + *cdev = tmp_dev; + *cdma_queue = tmp_q; + + return 0; +} + +/** + * dma_write - DMA write operation + * @dma_dev: DMA device pointer + * @rmt_seg: the remote segment pointer + * @local_seg: the local segment pointer + * @queue_id: DMA queue ID + * + * Invoke this interface to initiate a unilateral write operation request, + * sending the specified number of bytes of data from the designated local + * memory starting position to the specified destination address. + * Once the data is successfully written to the remote node, the application + * can poll the queue to obtain the completion message. + * + * Context: Process context. Takes and releases the spin_lock. + * Return: operation result, DMA_STATUS_OK on success + */ +enum dma_status dma_write(struct dma_device *dma_dev, struct dma_seg *rmt_seg, + struct dma_seg *local_seg, int queue_id) +{ + struct cdma_queue *cdma_queue = NULL; + struct cdma_dev *cdev = NULL; + int ret; + + if (!dma_dev || !rmt_seg || !local_seg) { + pr_err("write input parameters error.\n"); + return DMA_STATUS_INVAL; + } + + ret = cdma_param_transfer(dma_dev, queue_id, &cdev, &cdma_queue); + if (ret) + return DMA_STATUS_INVAL; + + ret = cdma_write(cdev, cdma_queue, local_seg, rmt_seg, NULL); + if (ret) + return DMA_STATUS_INVAL; + + return DMA_STATUS_OK; +} +EXPORT_SYMBOL_GPL(dma_write); + +/** + * dma_write_with_notify - DMA write with notify operation + * @dma_dev: DMA device pointer + * @rmt_seg: the remote segment pointer + * @local_seg: the local segment pointer + * @queue_id: DMA queue ID + * @data: notify data for write with notify operation + * + * Invoke this interface to initiate a write notify operation request for a + * unilateral operation, which sends a specified number of bytes of data from a + * designated starting position in local memory to a specified destination address. + * Once the data is successfully read from the remote node into local memory, + * the application can poll the queue to obtain the completion message. + * + * Context: Process context. Takes and releases the spin_lock. + * Return: operation result, DMA_STATUS_OK on success + */ +enum dma_status dma_write_with_notify(struct dma_device *dma_dev, + struct dma_seg *rmt_seg, + struct dma_seg *local_seg, int queue_id, + struct dma_notify_data *data) +{ + struct cdma_queue *cdma_queue = NULL; + struct cdma_dev *cdev = NULL; + int ret; + + if (!dma_dev || !rmt_seg || !local_seg || !data || !data->notify_seg) { + pr_err("write with notify input parameters error.\n"); + return DMA_STATUS_INVAL; + } + + ret = cdma_param_transfer(dma_dev, queue_id, &cdev, &cdma_queue); + if (ret) + return DMA_STATUS_INVAL; + + ret = cdma_write(cdev, cdma_queue, local_seg, rmt_seg, data); + if (ret) + return DMA_STATUS_INVAL; + + return DMA_STATUS_OK; +} +EXPORT_SYMBOL_GPL(dma_write_with_notify); + +/** + * dma_read - DMA read operation + * @dma_dev: DMA device pointer + * @rmt_seg: the remote segment pointer + * @local_seg: the local segment pointer + * @queue_id: DMA queue ID + * + * Invoke this interface to initiate a unidirectional read operation request, + * reading data from the specified remote address to the designated local cache + * starting position. + * Once the data is successfully read from the remote node to the local memory, + * the application can poll the queue to obtain the completion message. + * + * Context: Process context. Takes and releases the spin_lock. + * Return: operation result, DMA_STATUS_OK on success + */ +enum dma_status dma_read(struct dma_device *dma_dev, struct dma_seg *rmt_seg, + struct dma_seg *local_seg, int queue_id) +{ + struct cdma_queue *cdma_queue = NULL; + struct cdma_dev *cdev = NULL; + int ret; + + if (!dma_dev || !rmt_seg || !local_seg) { + pr_err("read input parameters error.\n"); + return DMA_STATUS_INVAL; + } + + ret = cdma_param_transfer(dma_dev, queue_id, &cdev, &cdma_queue); + if (ret) + return DMA_STATUS_INVAL; + + ret = cdma_read(cdev, cdma_queue, local_seg, rmt_seg); + if (ret) + return DMA_STATUS_INVAL; + + return DMA_STATUS_OK; +} +EXPORT_SYMBOL_GPL(dma_read); + +/** + * dma_cas - DMA cas operation + * @dma_dev: DMA device pointer + * @rmt_seg: the remote segment pointer + * @local_seg: the local segment pointer + * @queue_id: DMA queue ID + * @data: compare data and swap data for cas operaion + * + * Initiate a request for a unilateral atomic CAS operation. Once the operation + * is successful, the application can poll the queue to obtain the completion + * message. + * + * Context: Process context. Takes and releases the spin_lock. + * Return: operation result, DMA_STATUS_OK on success + */ +enum dma_status dma_cas(struct dma_device *dma_dev, struct dma_seg *rmt_seg, + struct dma_seg *local_seg, int queue_id, + struct dma_cas_data *data) +{ + struct cdma_queue *cdma_queue = NULL; + struct cdma_dev *cdev = NULL; + int ret; + + if (!dma_dev || !rmt_seg || !local_seg || !data) { + pr_err("cas input parameters error.\n"); + return DMA_STATUS_INVAL; + } + + ret = cdma_param_transfer(dma_dev, queue_id, &cdev, &cdma_queue); + if (ret) + return DMA_STATUS_INVAL; + + ret = cdma_cas(cdev, cdma_queue, local_seg, rmt_seg, data); + if (ret) + return DMA_STATUS_INVAL; + + return DMA_STATUS_OK; +} +EXPORT_SYMBOL_GPL(dma_cas); + +/** + * dma_faa - DMA faa operation + * @dma_dev: DMA device pointer + * @rmt_seg: the remote segment pointer + * @local_seg: the local segment pointer + * @queue_id: DMA queue ID + * @add: add data for faa operation + * + * Initiate a request for a unilateral atomic FAA operation. Once the operation + * is successful, the application can poll the queue to obtain the completion + * message. + * + * Context: Process context. Takes and releases the spin_lock. + * Return: operation result, DMA_STATUS_OK on success + */ +enum dma_status dma_faa(struct dma_device *dma_dev, struct dma_seg *rmt_seg, + struct dma_seg *local_seg, int queue_id, u64 add) +{ + struct cdma_queue *cdma_queue = NULL; + struct cdma_dev *cdev = NULL; + int ret; + + if (!dma_dev || !rmt_seg || !local_seg) { + pr_err("faa input parameters error.\n"); + return DMA_STATUS_INVAL; + } + + ret = cdma_param_transfer(dma_dev, queue_id, &cdev, &cdma_queue); + if (ret) + return DMA_STATUS_INVAL; + + ret = cdma_faa(cdev, cdma_queue, local_seg, rmt_seg, add); + if (ret) + return DMA_STATUS_INVAL; + + return DMA_STATUS_OK; +} +EXPORT_SYMBOL_GPL(dma_faa); + +/** + * dma_poll_queue - DMA polling queue + * @dma_dev: DMA device pointer + * @queue_id : DMA queue ID + * @cr_cnt: number of completion record + * @cr: completion record pointer + * + * Poll the DMA channel completion event, and the polling result is returned to + * the address specified by the parameter cr. + * The cr data structure includes information such as the result of the request + * execution, the length of data transferred, and the type of error. + * The caller must ensure that the number of parameters cr_cnt matches the number + * of addresses specified by cr. + * + * Context: Process context. + * Return: Polling operation results >0 on success, others on failed + */ +int dma_poll_queue(struct dma_device *dma_dev, int queue_id, u32 cr_cnt, + struct dma_cr *cr) +{ + struct cdma_queue *cdma_queue; + struct cdma_dev *cdev; + u32 eid; + + if (!dma_dev || !cr_cnt || !cr) { + pr_err("the poll queue input parameter is invalid.\n"); + return -EINVAL; + } + + eid = dma_dev->attr.eid.dw0; + cdev = get_cdma_dev_by_eid(eid); + if (!cdev) { + pr_err("get cdma dev failed, eid = 0x%x.\n", eid); + return -EINVAL; + } + + if (cdev->status == CDMA_SUSPEND) { + pr_warn("cdma device is not prepared, eid = 0x%x.\n", eid); + return -EINVAL; + } + + cdma_queue = cdma_find_queue(cdev, queue_id); + if (!cdma_queue || !cdma_queue->jfc) { + dev_err(cdev->dev, "get cdma queue failed, queue_id = %d.\n", + queue_id); + return -EINVAL; + } + + return cdma_poll_jfc(cdma_queue->jfc, cr_cnt, cr); +} +EXPORT_SYMBOL_GPL(dma_poll_queue); + +/** + * dma_register_client - DMA register client + * @client: DMA device client pointer + * + * Register the management software interface to notify the management software + * that the DMA driver is online. After loading or resetting and restarting the + * driver, call the add interface to notify the management software to request + * the resources required by DMA. When the driver is reset, deregistered, or + * unloaded, call the stop interface to notify the management software to stop + * using the DMA channel, and then call the remove interface to notify the + * management software to delete the DMA resources. + * + * Context: Process context. + * Return: operation result, 0 on success, others on failed + */ +int dma_register_client(struct dma_client *client) +{ + struct cdma_dev *cdev = NULL; + struct xarray *cdma_devs_tbl; + unsigned long index = 0; + u32 devs_num; + + if (client == NULL || client->client_name == NULL || + client->add == NULL || client->remove == NULL || + client->stop == NULL) { + pr_err("invalid parameter.\n"); + return -EINVAL; + } + + if (strnlen(client->client_name, DMA_MAX_DEV_NAME) >= DMA_MAX_DEV_NAME) { + pr_err("invalid parameter, client name.\n"); + return -EINVAL; + } + + down_write(&g_device_rwsem); + + cdma_devs_tbl = get_cdma_dev_tbl(&devs_num); + + xa_for_each(cdma_devs_tbl, index, cdev) { + if (client->add && client->add(cdev->eid)) + pr_info("dma client: %s add failed.\n", + client->client_name); + } + down_write(&g_clients_rwsem); + list_add_tail(&client->list_node, &g_client_list); + up_write(&g_clients_rwsem); + up_write(&g_device_rwsem); + + pr_info("dma client: %s register success.\n", client->client_name); + return 0; +} +EXPORT_SYMBOL_GPL(dma_register_client); + +/** + * dma_unregister_client - DMA unregister client + * @client: DMA device client pointer + * + * Unregister the management software interface, and delete client resources + * + * Context: Process context. + * Return: NA + */ +void dma_unregister_client(struct dma_client *client) +{ + struct cdma_dev *cdev = NULL; + struct xarray *cdma_devs_tbl; + unsigned long index = 0; + u32 devs_num; + + if (client == NULL || client->client_name == NULL || + client->add == NULL || client->remove == NULL || + client->stop == NULL) { + pr_err("Invalid parameter.\n"); + return; + } + + if (strnlen(client->client_name, DMA_MAX_DEV_NAME) >= DMA_MAX_DEV_NAME) { + pr_err("invalid parameter, client name.\n"); + return; + } + + down_write(&g_device_rwsem); + cdma_devs_tbl = get_cdma_dev_tbl(&devs_num); + + xa_for_each(cdma_devs_tbl, index, cdev) { + if (client->stop && client->remove) { + client->stop(cdev->eid); + client->remove(cdev->eid); + } + } + + down_write(&g_clients_rwsem); + list_del(&client->list_node); + up_write(&g_clients_rwsem); + up_write(&g_device_rwsem); + + pr_info("dma client: %s unregister success.\n", client->client_name); +} +EXPORT_SYMBOL_GPL(dma_unregister_client); diff --git a/drivers/ub/cdma/cdma_chardev.c b/drivers/ub/cdma/cdma_chardev.c new file mode 100644 index 0000000000000000000000000000000000000000..3614609d683eec8ef9b41738c072658d1386e89c --- /dev/null +++ b/drivers/ub/cdma/cdma_chardev.c @@ -0,0 +1,419 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define pr_fmt(fmt) "CDMA: " fmt +#define dev_fmt pr_fmt + +#include +#include +#include "cdma_cmd.h" +#include "cdma_ioctl.h" +#include "cdma_context.h" +#include "cdma_chardev.h" +#include "cdma_jfs.h" +#include "cdma_types.h" +#include "cdma_uobj.h" +#include "cdma.h" +#include "cdma_mmap.h" + +#define CDMA_DEVICE_NAME "cdma/dev" + +struct cdma_num_manager { + struct idr idr; + spinlock_t lock; +}; + +static struct cdma_num_manager cdma_num_mg = { + .idr = IDR_INIT(cdma_num_mg.idr), + .lock = __SPIN_LOCK_UNLOCKED(cdma_num_mg.lock), +}; + +static void cdma_num_free(struct cdma_dev *cdev) +{ + spin_lock(&cdma_num_mg.lock); + idr_remove(&cdma_num_mg.idr, cdev->chardev.dev_num); + spin_unlock(&cdma_num_mg.lock); +} + +static inline u64 cdma_get_mmap_idx(struct vm_area_struct *vma) +{ + return (vma->vm_pgoff >> MAP_INDEX_SHIFT) & MAP_INDEX_MASK; +} + +static inline int cdma_get_mmap_cmd(struct vm_area_struct *vma) +{ + return (vma->vm_pgoff & MAP_COMMAND_MASK); +} + +static int cdma_num_alloc(struct cdma_dev *cdev) +{ +#define CDMA_START 0 +#define CDMA_END 0xffff + int id; + + idr_preload(GFP_KERNEL); + spin_lock(&cdma_num_mg.lock); + id = idr_alloc(&cdma_num_mg.idr, cdev, CDMA_START, CDMA_END, GFP_NOWAIT); + spin_unlock(&cdma_num_mg.lock); + idr_preload_end(); + + return id; +} + +static long cdma_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ +#define CDMA_MAX_CMD_SIZE 8192 + struct cdma_file *cfile = (struct cdma_file *)file->private_data; + struct cdma_ioctl_hdr hdr = { 0 }; + int ret; + + if (!cfile->cdev || cfile->cdev->status == CDMA_SUSPEND) { + pr_info("ioctl cdev is invalid.\n"); + return -ENODEV; + } + cdma_cmd_inc(cfile->cdev); + + if (cmd == CDMA_SYNC) { + ret = copy_from_user(&hdr, (void *)arg, sizeof(hdr)); + if (ret || hdr.args_len > CDMA_MAX_CMD_SIZE) { + pr_err("copy user ret = %d, input parameter len = %u.\n", + ret, hdr.args_len); + cdma_cmd_dec(cfile->cdev); + return -EINVAL; + } + ret = cdma_cmd_parse(cfile, &hdr); + cdma_cmd_dec(cfile->cdev); + return ret; + } + + pr_err("invalid ioctl command, command = %u.\n", cmd); + cdma_cmd_dec(cfile->cdev); + return -ENOIOCTLCMD; +} + +static int cdma_remap_check_jfs_id(struct cdma_file *cfile, u32 jfs_id) +{ + struct cdma_dev *cdev = cfile->cdev; + struct cdma_jfs *jfs; + int ret = -EINVAL; + + spin_lock(&cdev->jfs_table.lock); + jfs = idr_find(&cdev->jfs_table.idr_tbl.idr, jfs_id); + if (!jfs) { + spin_unlock(&cdev->jfs_table.lock); + dev_err(cdev->dev, + "check failed, jfs_id = %u not exist.\n", jfs_id); + return ret; + } + + if (cfile->uctx != jfs->base_jfs.ctx) { + dev_err(cdev->dev, + "check failed, jfs_id = %u, uctx invalid\n", jfs_id); + spin_unlock(&cdev->jfs_table.lock); + return -EINVAL; + } + spin_unlock(&cdev->jfs_table.lock); + + return 0; +} + +static int cdma_remap_pfn_range(struct cdma_file *cfile, struct vm_area_struct *vma) +{ +#define JFC_DB_UNMAP_BOUND 1 + struct cdma_dev *cdev = cfile->cdev; + resource_size_t db_addr; + u64 address; + u32 jfs_id; + u32 cmd; + + if (cdev->status == CDMA_SUSPEND) { + dev_warn(cdev->dev, "cdev is resetting.\n"); + return -EBUSY; + } + + db_addr = cdev->db_base; + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + cmd = cdma_get_mmap_cmd(vma); + switch (cmd) { + case CDMA_MMAP_JFC_PAGE: + if (io_remap_pfn_range(vma, vma->vm_start, + jfc_arm_mode > JFC_DB_UNMAP_BOUND ? + (uint64_t)db_addr >> PAGE_SHIFT : + page_to_pfn(cdev->arm_db_page), + PAGE_SIZE, vma->vm_page_prot)) { + dev_err(cdev->dev, "remap jfc page fail.\n"); + return -EAGAIN; + } + break; + case CDMA_MMAP_JETTY_DSQE: + jfs_id = cdma_get_mmap_idx(vma); + if (cdma_remap_check_jfs_id(cfile, jfs_id)) { + dev_err(cdev->dev, + "mmap failed, invalid jfs_id = %u\n", jfs_id); + return -EINVAL; + } + + address = (uint64_t)db_addr + CDMA_JETTY_DSQE_OFFSET + jfs_id * PAGE_SIZE; + + if (io_remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT, + PAGE_SIZE, vma->vm_page_prot)) { + dev_err(cdev->dev, "remap jetty page failed.\n"); + return -EAGAIN; + } + break; + default: + dev_err(cdev->dev, + "mmap failed, cmd(%u) is not supported.\n", cmd); + return -EINVAL; + } + + return 0; +} + +static int cdma_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct cdma_file *cfile = (struct cdma_file *)file->private_data; + struct cdma_umap_priv *priv; + int ret; + + if (!cfile->cdev || cfile->cdev->status == CDMA_SUSPEND) { + pr_info("mmap cdev is invalid.\n"); + return -ENODEV; + } + + if (((vma->vm_end - vma->vm_start) % PAGE_SIZE) != 0) { + pr_err("mmap failed, expect vm area size to be an integer multiple of page size.\n"); + return -EINVAL; + } + + priv = kzalloc(sizeof(struct cdma_umap_priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + vma->vm_ops = cdma_get_umap_ops(); + vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_WIPEONFORK | VM_IO); + + mutex_lock(&cfile->ctx_mutex); + ret = cdma_remap_pfn_range(cfile, vma); + if (ret) { + mutex_unlock(&cfile->ctx_mutex); + kfree(priv); + return ret; + } + mutex_unlock(&cfile->ctx_mutex); + + cdma_umap_priv_init(priv, vma); + + return 0; +} + +static void cdma_mmu_release(struct mmu_notifier *mn, struct mm_struct *mm) +{ + struct cdma_mn *mn_notifier = container_of(mn, struct cdma_mn, mn); + struct cdma_file *cfile = container_of(mn_notifier, struct cdma_file, mn_notifier); + + if (mn_notifier->mm != mm || mn_notifier->mm == NULL) { + pr_info("mm already released.\n"); + return; + } + mn_notifier->mm = NULL; + + mutex_lock(&cfile->ctx_mutex); + cdma_cleanup_context_uobj(cfile, CDMA_REMOVE_CLOSE); + if (cfile->uctx) + cdma_cleanup_context_res(cfile->uctx); + cfile->uctx = NULL; + mutex_unlock(&cfile->ctx_mutex); +} + +static const struct mmu_notifier_ops cdma_mm_notifier_ops = { + .release = cdma_mmu_release +}; + +static int cdma_register_mmu(struct cdma_file *file) +{ + struct cdma_mn *mn_notifier = &file->mn_notifier; + int ret; + + mn_notifier->mm = current->mm; + mn_notifier->mn.ops = &cdma_mm_notifier_ops; + ret = mmu_notifier_register(&mn_notifier->mn, current->mm); + if (ret) + mn_notifier->mm = NULL; + + return ret; +} + +static void cdma_unregister_mmu(struct cdma_file *cfile) +{ + struct cdma_mn *mn_notifier = &cfile->mn_notifier; + struct mm_struct *mm = mn_notifier->mm; + + if (!mm) + return; + + cfile->mn_notifier.mm = NULL; + mmu_notifier_unregister(&cfile->mn_notifier.mn, mm); +} + +static int cdma_open(struct inode *inode, struct file *file) +{ + struct cdma_chardev *chardev; + struct cdma_file *cfile; + struct cdma_dev *cdev; + int ret; + + chardev = container_of(inode->i_cdev, struct cdma_chardev, cdev); + cdev = container_of(chardev, struct cdma_dev, chardev); + + if (cdev->status == CDMA_SUSPEND) { + dev_warn(cdev->dev, "cdev is resetting.\n"); + return -EBUSY; + } + + cfile = kzalloc(sizeof(struct cdma_file), GFP_KERNEL); + if (!cfile) + return -ENOMEM; + + ret = cdma_register_mmu(cfile); + if (ret) { + dev_err(cdev->dev, "register mmu failed, ret = %d.\n", ret); + kfree(cfile); + return ret; + } + + cdma_init_uobj_idr(cfile); + mutex_lock(&cdev->file_mutex); + cfile->cdev = cdev; + cfile->uctx = NULL; + kref_init(&cfile->ref); + file->private_data = cfile; + mutex_init(&cfile->ctx_mutex); + list_add_tail(&cfile->list, &cdev->file_list); + mutex_init(&cfile->umap_mutex); + INIT_LIST_HEAD(&cfile->umaps_list); + nonseekable_open(inode, file); + mutex_unlock(&cdev->file_mutex); + + return 0; +} + +static int cdma_close(struct inode *inode, struct file *file) +{ + struct cdma_file *cfile = (struct cdma_file *)file->private_data; + struct cdma_dev *cdev; + + mutex_lock(&g_cdma_reset_mutex); + + cdev = cfile->cdev; + if (!cdev) { + mutex_unlock(&g_cdma_reset_mutex); + kref_put(&cfile->ref, cdma_release_file); + inode->i_cdev = NULL; + return 0; + } + + mutex_lock(&cdev->file_mutex); + list_del(&cfile->list); + mutex_unlock(&cdev->file_mutex); + + mutex_lock(&cfile->ctx_mutex); + cdma_cleanup_context_uobj(cfile, CDMA_REMOVE_CLOSE); + if (cfile->uctx) + cdma_cleanup_context_res(cfile->uctx); + cfile->uctx = NULL; + + mutex_unlock(&cfile->ctx_mutex); + mutex_unlock(&g_cdma_reset_mutex); + kref_put(&cfile->ref, cdma_release_file); + pr_debug("cdma close success.\n"); + + return 0; +} + +static const struct file_operations cdma_ops = { + .owner = THIS_MODULE, + .unlocked_ioctl = cdma_ioctl, + .mmap = cdma_mmap, + .open = cdma_open, + .release = cdma_close, +}; + +void cdma_destroy_chardev(struct cdma_dev *cdev) +{ + struct cdma_chardev *chardev = &cdev->chardev; + + if (!chardev->dev) + return; + + device_destroy(cdma_cdev_class, chardev->devno); + cdev_del(&chardev->cdev); + unregister_chrdev_region(chardev->devno, CDMA_MAX_DEVICES); + cdma_num_free(cdev); +} + +int cdma_create_chardev(struct cdma_dev *cdev) +{ + struct cdma_chardev *chardev = &cdev->chardev; + int ret; + + chardev->dev_num = cdma_num_alloc(cdev); + if (chardev->dev_num < 0) { + dev_err(cdev->dev, "alloc dev_num failed, ret = %d\n", chardev->dev_num); + return -ENOMEM; + } + + ret = snprintf(chardev->name, sizeof(chardev->name), + "%s.%d", CDMA_DEVICE_NAME, chardev->dev_num); + if (ret < 0) { + dev_err(cdev->dev, "sprintf failed in create cdma chardev\n"); + goto num_free; + } + + ret = alloc_chrdev_region(&chardev->devno, 0, CDMA_MAX_DEVICES, + chardev->name); + if (ret) { + dev_err(cdev->dev, "alloc chrdev region failed, ret = %d\n", ret); + goto num_free; + } + + cdev_init(&chardev->cdev, &cdma_ops); + ret = cdev_add(&chardev->cdev, chardev->devno, CDMA_MAX_DEVICES); + if (ret) { + dev_err(cdev->dev, "cdev add failed, ret = %d\n", ret); + goto chrdev_unregister; + } + + chardev->dev = device_create(cdma_cdev_class, NULL, chardev->devno, + NULL, chardev->name); + if (IS_ERR(chardev->dev)) { + ret = PTR_ERR(chardev->dev); + dev_err(cdev->dev, "create device failed, ret = %d\n", ret); + goto cdev_delete; + } + + dev_dbg(cdev->dev, "create chardev: %s succeeded\n", chardev->name); + return 0; + +cdev_delete: + cdev_del(&chardev->cdev); +chrdev_unregister: + unregister_chrdev_region(chardev->devno, CDMA_MAX_DEVICES); +num_free: + cdma_num_free(cdev); + return ret; +} + +void cdma_release_file(struct kref *ref) +{ + struct cdma_file *cfile = container_of(ref, struct cdma_file, ref); + + if (cfile->fault_page) + __free_pages(cfile->fault_page, 0); + cdma_unregister_mmu(cfile); + mutex_destroy(&cfile->umap_mutex); + mutex_destroy(&cfile->ctx_mutex); + idr_destroy(&cfile->idr); + kfree(cfile); +} diff --git a/drivers/ub/cdma/cdma_chardev.h b/drivers/ub/cdma/cdma_chardev.h new file mode 100644 index 0000000000000000000000000000000000000000..0bd4fcc654ff4d77a751dfe7a6600e4bb83dced5 --- /dev/null +++ b/drivers/ub/cdma/cdma_chardev.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_CHARDEV_H__ +#define __CDMA_CHARDEV_H__ + +#include + +#define CDMA_TEST_NAME "cdma_dev" +#define CDMA_MAX_DEVICES 1 +#define CDMA_JETTY_DSQE_OFFSET 0x1000 + +extern struct class *cdma_cdev_class; + +struct cdma_dev; + +void cdma_destroy_chardev(struct cdma_dev *cdev); +int cdma_create_chardev(struct cdma_dev *cdev); +void cdma_release_file(struct kref *ref); + +#endif /* _CDMA_CHARDEV_H_ */ diff --git a/drivers/ub/cdma/cdma_cmd.c b/drivers/ub/cdma/cdma_cmd.c new file mode 100644 index 0000000000000000000000000000000000000000..c8bf01d930adc12d63c4fd7d2cf4253fc0050a2d --- /dev/null +++ b/drivers/ub/cdma/cdma_cmd.c @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "CDMA: " fmt + +#include +#include + +#include "cdma.h" +#include +#include +#include +#include "cdma_cmd.h" + +static int cdma_cmd_query_fw_resource(struct cdma_dev *cdev, struct cdma_ue_info *out_addr) +{ +#define CDMA_QUERY_UE_RES 0x0004 + struct ubase_cmd_buf out = { 0 }; + struct ubase_cmd_buf in = { 0 }; + + ubase_fill_inout_buf(&in, CDMA_QUERY_UE_RES, true, 0, NULL); + ubase_fill_inout_buf(&out, CDMA_QUERY_UE_RES, true, + sizeof(*out_addr), out_addr); + + return ubase_cmd_send_inout(cdev->adev, &in, &out); +} + +static int cdma_query_caps_from_firmware(struct cdma_dev *cdev) +{ + struct cdma_caps *caps = &cdev->caps; + struct cdma_ue_info cmd = { 0 }; + int ret; + + ret = cdma_cmd_query_fw_resource(cdev, &cmd); + if (ret) + return dev_err_probe(cdev->dev, ret, "query fw resource failed\n"); + + caps->jfs_sge = cmd.jfs_sge; + caps->trans_mode = cmd.trans_mode; + caps->seid.max_cnt = cmd.seid_upi_tbl_num; + caps->feature = cmd.cap_info; + caps->ue_cnt = cmd.ue_cnt; + caps->ue_id = cmd.ue_id; + + dev_dbg(cdev->dev, "jfs_sge = 0x%x, trans_mode = 0x%x, seid.max_cnt = 0x%x\n", + caps->jfs_sge, caps->trans_mode, caps->seid.max_cnt); + dev_dbg(cdev->dev, "feature = 0x%x, ue_cnt = 0x%x, ue_id = 0x%x\n", + caps->feature, caps->ue_cnt, caps->ue_id); + + return 0; +} + +static int cdma_set_caps_from_adev_caps(struct cdma_dev *cdev) +{ +#define MAX_WQEBB_IN_SQE 4 + struct cdma_caps *caps = &cdev->caps; + struct ubase_adev_caps *adev_caps; + + adev_caps = ubase_get_cdma_caps(cdev->adev); + if (!adev_caps) { + dev_err(cdev->dev, "get cdma adev caps failed\n"); + return -EINVAL; + } + + caps->jfs.max_cnt = adev_caps->jfs.max_cnt; + caps->jfs.depth = adev_caps->jfs.depth / MAX_WQEBB_IN_SQE; + caps->jfs.start_idx = adev_caps->jfs.start_idx; + caps->jfc.max_cnt = adev_caps->jfc.max_cnt; + caps->jfc.depth = adev_caps->jfc.depth; + caps->jfc.start_idx = adev_caps->jfc.start_idx; + caps->cqe_size = adev_caps->cqe_size; + + return 0; +} + +static int cdma_set_caps_from_ubase_caps(struct cdma_dev *cdev) +{ + struct cdma_caps *caps = &cdev->caps; + struct ubase_caps *ubase_caps; + + ubase_caps = ubase_get_dev_caps(cdev->adev); + if (!ubase_caps) { + dev_err(cdev->dev, "get cdma ubase caps failed\n"); + return -EINVAL; + } + + caps->comp_vector_cnt = ubase_caps->num_ceq_vectors; + caps->public_jetty_cnt = ubase_caps->public_jetty_cnt; + cdev->eid = ubase_caps->eid; + cdev->upi = ubase_caps->upi; + + return 0; +} + +static int cdma_set_caps_from_adev_qos(struct cdma_dev *cdev) +{ + struct ubase_adev_qos *qos; + + qos = ubase_get_adev_qos(cdev->adev); + if (!qos) { + dev_err(cdev->dev, "get cdma adev qos failed\n"); + return -EINVAL; + } + + if (!qos->ctp_sl_num || qos->ctp_sl_num > CDMA_MAX_SL_NUM) { + dev_err(cdev->dev, "sl num %u is invalid\n", + qos->ctp_sl_num); + return -EINVAL; + } + + cdev->sl_num = qos->ctp_sl_num; + memcpy(cdev->sl, qos->ctp_sl, qos->ctp_sl_num); + + return 0; +} + +int cdma_init_dev_caps(struct cdma_dev *cdev) +{ + struct cdma_caps *caps = &cdev->caps; + int ret; + u8 i; + + ret = cdma_query_caps_from_firmware(cdev); + if (ret) + return ret; + + ret = cdma_set_caps_from_adev_caps(cdev); + if (ret) + return ret; + + ret = cdma_set_caps_from_ubase_caps(cdev); + if (ret) + return ret; + + ret = cdma_set_caps_from_adev_qos(cdev); + if (ret) + return ret; + + caps->queue.max_cnt = min(caps->jfs.max_cnt, caps->jfc.max_cnt); + caps->queue.start_idx = 0; + caps->jfce.max_cnt = caps->jfc.max_cnt; + caps->jfce.start_idx = 0; + + dev_info(cdev->dev, "query cdev eid = 0x%x, cdev upi = 0x%x\n", cdev->eid, + cdev->upi); + dev_info(cdev->dev, "queue:max_cnt = 0x%x, start_idx = 0x%x\n", + caps->queue.max_cnt, caps->queue.start_idx); + dev_info(cdev->dev, "jfs:max_cnt = 0x%x, depth = 0x%x, start_idx = 0x%x\n", + caps->jfs.max_cnt, caps->jfs.depth, caps->jfs.start_idx); + dev_info(cdev->dev, "jfce:max_cnt = 0x%x, depth = 0x%x, start_idx = 0x%x\n", + caps->jfce.max_cnt, caps->jfce.depth, caps->jfce.start_idx); + dev_info(cdev->dev, "jfc:max_cnt = 0x%x, depth = 0x%x, start_idx = 0x%x\n", + caps->jfc.max_cnt, caps->jfc.depth, caps->jfc.start_idx); + dev_info(cdev->dev, "comp_vector_cnt = 0x%x, public_jetty_cnt = 0x%x\n", + caps->comp_vector_cnt, caps->public_jetty_cnt); + dev_info(cdev->dev, "sl_num = 0x%x\n", cdev->sl_num); + for (i = 0; i < cdev->sl_num; i++) + dev_info(cdev->dev, "sl[%u] = 0x%x\n", i, cdev->sl[i]); + + return 0; +} + +int cdma_ctrlq_query_eu(struct cdma_dev *cdev) +{ +#define CDMA_CTRLQ_QUERY_SEID_UPI 0x1 +#define CDMA_CTRLQ_CMD_SEID_UPI 0xB5 + struct cdma_device_attr *attr = &cdev->base.attr; + struct eu_query_out out_query = { 0 }; + struct eu_query_in in_query = { 0 }; + struct ubase_ctrlq_msg msg = { 0 }; + struct eu_info *eus = attr->eus; + int ret; + u8 i; + + in_query.cmd = CDMA_CTRLQ_CMD_SEID_UPI; + + msg = (struct ubase_ctrlq_msg) { + .service_ver = UBASE_CTRLQ_SER_VER_01, + .service_type = UBASE_CTRLQ_SER_TYPE_DEV_REGISTER, + .opcode = CDMA_CTRLQ_QUERY_SEID_UPI, + .need_resp = 1, + .is_resp = 0, + .resv = 0, + .resp_seq = 0, + .in_size = sizeof(in_query), + .in = &in_query, + .out_size = sizeof(out_query), + .out = &out_query, + }; + + ret = ubase_ctrlq_send_msg(cdev->adev, &msg); + if (ret) { + dev_err(cdev->dev, + "query seid upi from ctrl cpu failed, ret = %d.\n", ret); + return ret; + } + + if (!out_query.seid_num || out_query.seid_num > CDMA_MAX_EU_NUM) { + dev_err(cdev->dev, + "query seid upi num is invalid, num = %u.\n", + out_query.seid_num); + return -EINVAL; + } + + mutex_lock(&cdev->eu_mutex); + memcpy(eus, out_query.eus, sizeof(struct eu_info) * out_query.seid_num); + attr->eu_num = out_query.seid_num; + + for (i = 0; i < attr->eu_num; i++) + dev_dbg(cdev->dev, + "cdma init eus[%u], upi = 0x%x, eid = 0x%x, eid_idx = 0x%x.\n", + i, eus[i].upi, eus[i].eid.dw0, eus[i].eid_idx); + mutex_unlock(&cdev->eu_mutex); + + return 0; +} + +void cdma_cmd_inc(struct cdma_dev *cdev) +{ + atomic_inc(&cdev->cmdcnt); +} + +void cdma_cmd_dec(struct cdma_dev *cdev) +{ + if (atomic_dec_and_test(&cdev->cmdcnt)) + complete(&cdev->cmddone); +} + +void cdma_cmd_flush(struct cdma_dev *cdev) +{ + cdma_cmd_dec(cdev); + pr_info("cmd flush cmdcnt is %d\n", atomic_read(&cdev->cmdcnt)); + wait_for_completion(&cdev->cmddone); +} diff --git a/drivers/ub/cdma/cdma_cmd.h b/drivers/ub/cdma/cdma_cmd.h new file mode 100644 index 0000000000000000000000000000000000000000..f85331c8c51b7f65308a3ee1af7b273f17f2e7a9 --- /dev/null +++ b/drivers/ub/cdma/cdma_cmd.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_CMD_H__ +#define __CDMA_CMD_H__ + +#include + +struct cdma_dev; + +struct cdma_ue_info { + /* BD0 */ + u16 jfs_num_shift : 4; + u16 jfr_num_shift : 4; + u16 jfc_num_shift : 4; + u16 jetty_num_shift : 4; + + u16 jetty_grp_num; + + u16 jfs_depth_shift : 4; + u16 jfr_depth_shift : 4; + u16 jfc_depth_shift : 4; + u16 cqe_size_shift : 4; + + u16 jfs_sge : 5; + u16 jfr_sge : 5; + u16 jfs_rsge : 6; + + u16 max_jfs_inline_sz; + u16 max_jfc_inline_sz; + u32 cap_info; + + u16 trans_mode : 5; + u16 ue_num : 8; + u16 virtualization : 1; + u16 rsvd0 : 2; + + u16 ue_cnt; + u8 ue_id; + u8 default_cong_alg; + u8 cons_ctrl_alg; + u8 cc_priority_cnt; + /* BD1 */ + u16 src_addr_tbl_sz; + u16 src_addr_tbl_num; + u16 dest_addr_tbl_sz; + u16 dest_addr_tbl_num; + u16 seid_upi_tbl_sz; + u16 seid_upi_tbl_num; + u16 tpm_tbl_sz; + u16 tpm_tbl_num; + u32 tp_range; + u8 port_num; + u8 port_id; + u8 rsvd1[2]; + u16 rc_queue_num; + u16 rc_depth; + u8 rc_entry; + u8 rsvd2[3]; + /* BD2 */ + u32 rsvd3[8]; + /* BD3 */ + u32 rsvd4[8]; +}; + +struct eu_query_in { + u32 cmd : 8; + u32 rsv : 24; +}; + +struct eu_query_out { + u32 seid_num : 8; + u32 rsv : 24; + struct eu_info eus[CDMA_MAX_EU_NUM]; +}; + +int cdma_init_dev_caps(struct cdma_dev *cdev); +int cdma_ctrlq_query_eu(struct cdma_dev *cdev); +void cdma_cmd_inc(struct cdma_dev *cdev); +void cdma_cmd_dec(struct cdma_dev *cdev); +void cdma_cmd_flush(struct cdma_dev *cdev); +#endif diff --git a/drivers/ub/cdma/cdma_common.c b/drivers/ub/cdma/cdma_common.c new file mode 100644 index 0000000000000000000000000000000000000000..62ae5f8be48c424613040ce13896bf755398e400 --- /dev/null +++ b/drivers/ub/cdma/cdma_common.c @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "CDMA: " fmt + +#include +#include +#include +#include "cdma_common.h" +#include "cdma.h" + +static inline void cdma_fill_umem(struct cdma_umem *umem, + struct cdma_umem_param *param) +{ + umem->dev = param->dev; + umem->va = param->va; + umem->length = param->len; + umem->flag = param->flag; +} + +static int cdma_pin_part_of_pages(u64 cur_base, u64 npages, u32 gup_flags, + struct page **page_list) +{ + /* + * page_list size is 4kB, the nr_pages should not larger than + * PAGE_SIZE / sizeof(struct page *) + */ + return pin_user_pages_fast(cur_base, + min_t(unsigned long, (unsigned long)npages, + PAGE_SIZE / sizeof(struct page *)), + gup_flags | FOLL_LONGTERM, page_list); +} + +static struct scatterlist *cdma_sg_set_page(struct scatterlist *sg_start, + int pinned, struct page **page_list) +{ + struct scatterlist *sg; + int i; + + for_each_sg(sg_start, sg, pinned, i) + sg_set_page(sg, page_list[i], PAGE_SIZE, 0); + + return sg; +} + +static u64 cdma_pin_pages(struct cdma_dev *cdev, struct cdma_umem *umem, + u64 npages, u32 gup_flags, struct page **pages) +{ + struct scatterlist *sg_list_start = umem->sg_head.sgl; + u64 cur_base = umem->va & PAGE_MASK; + u64 page_count = npages; + int pinned; + + while (page_count > 0) { + cond_resched(); + + pinned = cdma_pin_part_of_pages(cur_base, page_count, gup_flags, + pages); + if (pinned <= 0) { + dev_err(cdev->dev, + "pin pages failed, page_count = 0x%llx, pinned = %d.\n", + page_count, pinned); + return npages - page_count; + } + cur_base += (u64)pinned * PAGE_SIZE; + page_count -= (u64)pinned; + sg_list_start = cdma_sg_set_page(sg_list_start, pinned, pages); + } + + return npages; +} + +static u64 cdma_k_pin_pages(struct cdma_dev *cdev, struct cdma_umem *umem, + u64 npages) +{ + struct scatterlist *sg_cur = umem->sg_head.sgl; + u64 cur_base = umem->va & PAGE_MASK; + struct page *pg; + u64 n; + + for (n = 0; n < npages; n++) { + if (is_vmalloc_addr((void *)(uintptr_t)cur_base)) + pg = vmalloc_to_page((void *)(uintptr_t)cur_base); + else + pg = kmap_to_page((void *)(uintptr_t)cur_base); + + if (!pg) { + dev_err(cdev->dev, "vmalloc or kmap to page failed.\n"); + break; + } + + get_page(pg); + + cur_base += PAGE_SIZE; + + sg_set_page(sg_cur, pg, PAGE_SIZE, 0); + sg_cur = sg_next(sg_cur); + } + + return n; +} + +static void cdma_unpin_pages(struct cdma_umem *umem, u64 nents, bool is_kernel) +{ + struct scatterlist *sg; + struct page *page; + u32 i; + + for_each_sg(umem->sg_head.sgl, sg, nents, i) { + page = sg_page(sg); + + if (is_kernel) + put_page(page); + else + unpin_user_page(page); + } +} + +static struct cdma_umem *cdma_get_target_umem(struct cdma_umem_param *param, + struct page **page_list) +{ + struct cdma_dev *cdev = param->dev; + struct cdma_umem *umem; + u64 npages, pinned; + u32 gup_flags; + int ret = 0; + + umem = kzalloc(sizeof(*umem), GFP_KERNEL); + if (!umem) { + ret = -ENOMEM; + goto out; + } + + cdma_fill_umem(umem, param); + + npages = cdma_cal_npages(umem->va, umem->length); + if (!npages || npages > UINT_MAX) { + dev_err(cdev->dev, + "invalid npages %llu in getting target umem process.\n", npages); + ret = -EINVAL; + goto umem_kfree; + } + + ret = sg_alloc_table(&umem->sg_head, (unsigned int)npages, GFP_KERNEL); + if (ret) + goto umem_kfree; + + if (param->is_kernel) { + pinned = cdma_k_pin_pages(cdev, umem, npages); + } else { + gup_flags = param->flag.bs.writable ? FOLL_WRITE : 0; + pinned = cdma_pin_pages(cdev, umem, npages, gup_flags, + page_list); + } + + if (pinned != npages) { + ret = -ENOMEM; + goto umem_release; + } + + goto out; + +umem_release: + cdma_unpin_pages(umem, pinned, param->is_kernel); + sg_free_table(&umem->sg_head); +umem_kfree: + kfree(umem); +out: + return ret != 0 ? ERR_PTR(ret) : umem; +} + +static int cdma_verify_input(struct cdma_dev *cdev, u64 va, u64 len) +{ + if (((va + len) <= va) || PAGE_ALIGN(va + len) < (va + len)) { + dev_err(cdev->dev, "invalid address parameter, len = %llu.\n", + len); + return -EINVAL; + } + return 0; +} + +struct cdma_umem *cdma_umem_get(struct cdma_dev *cdev, u64 va, u64 len, + bool is_kernel) +{ + struct cdma_umem_param param; + struct page **page_list; + struct cdma_umem *umem; + int ret; + + ret = cdma_verify_input(cdev, va, len); + if (ret) + return ERR_PTR(ret); + + page_list = (struct page **)__get_free_page(GFP_KERNEL); + if (!page_list) + return ERR_PTR(-ENOMEM); + + param.dev = cdev; + param.va = va; + param.len = len; + param.flag.bs.writable = true; + param.flag.bs.non_pin = 0; + param.is_kernel = is_kernel; + umem = cdma_get_target_umem(¶m, page_list); + if (IS_ERR(umem)) + dev_err(cdev->dev, "get target umem failed.\n"); + + free_page((unsigned long)(uintptr_t)page_list); + return umem; +} + +void cdma_umem_release(struct cdma_umem *umem, bool is_kernel) +{ + if (IS_ERR_OR_NULL(umem)) + return; + + cdma_unpin_pages(umem, umem->sg_head.nents, is_kernel); + sg_free_table(&umem->sg_head); + kfree(umem); +} + +int cdma_k_alloc_buf(struct cdma_dev *cdev, size_t memory_size, + struct cdma_buf *buf) +{ + size_t aligned_memory_size; + int ret; + + aligned_memory_size = memory_size + CDMA_HW_PAGE_SIZE - 1; + buf->aligned_va = vmalloc(aligned_memory_size); + if (!buf->aligned_va) { + dev_err(cdev->dev, + "vmalloc kernel buf failed, size = %lu.\n", + aligned_memory_size); + return -ENOMEM; + } + + memset(buf->aligned_va, 0, aligned_memory_size); + buf->umem = cdma_umem_get(cdev, (u64)buf->aligned_va, + aligned_memory_size, true); + if (IS_ERR(buf->umem)) { + ret = PTR_ERR(buf->umem); + vfree(buf->aligned_va); + dev_err(cdev->dev, "pin kernel buf failed, ret = %d.\n", ret); + return ret; + } + + buf->addr = ((u64)buf->aligned_va + CDMA_HW_PAGE_SIZE - 1) & + ~(CDMA_HW_PAGE_SIZE - 1); + buf->kva = (void *)buf->addr; + + return 0; +} + +void cdma_k_free_buf(struct cdma_dev *cdev, size_t memory_size, + struct cdma_buf *buf) +{ + cdma_umem_release(buf->umem, true); + vfree(buf->aligned_va); + buf->aligned_va = NULL; + buf->kva = NULL; + buf->addr = 0; +} + +int cdma_pin_queue_addr(struct cdma_dev *cdev, u64 addr, u32 len, + struct cdma_buf *buf) +{ + int ret = 0; + + if (IS_ERR_OR_NULL(buf)) + return -EINVAL; + + buf->umem = cdma_umem_get(cdev, addr, len, false); + if (IS_ERR(buf->umem)) { + dev_err(cdev->dev, "get umem failed.\n"); + ret = PTR_ERR(buf->umem); + return ret; + } + + buf->addr = addr; + + return ret; +} + +void cdma_unpin_queue_addr(struct cdma_umem *umem) +{ + cdma_umem_release(umem, false); +} diff --git a/drivers/ub/cdma/cdma_common.h b/drivers/ub/cdma/cdma_common.h new file mode 100644 index 0000000000000000000000000000000000000000..58855991647dca5356c215029dfca3b793d2366c --- /dev/null +++ b/drivers/ub/cdma/cdma_common.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_COMMON_H__ +#define __CDMA_COMMON_H__ + +#include +#include "cdma.h" + +#define JETTY_DSQE_OFFSET 0x1000 +#define CDMA_USER_DATA_H_OFFSET 32U + +#define SQE_TOKEN_ID_L_MASK GENMASK(11, 0) +#define SQE_TOKEN_ID_H_OFFSET 12U +#define SQE_TOKEN_ID_H_MASK GENMASK(7, 0) +#define SQE_VA_L_OFFSET 12U +#define SQE_VA_L_VALID_BIT GENMASK(19, 0) +#define SQE_VA_H_OFFSET 32U +#define SQE_VA_H_VALID_BIT GENMASK(31, 0) +#define WQE_BB_SIZE_SHIFT 6 +#define AVAIL_SGMT_OST_INIT 512 + +#define CDMA_RANGE_INDEX_ENTRY_CNT 0x100000 +#define CDMA_SEGMENT_ENTRY_CNT 0x10000 + +#define CDMA_ENABLE_FLAG 1 +#define CDMA_DISABLE_FLAG 0 + +#define CDMA_DB_SIZE 64 + +#define CDMA_ATOMIC_LEN_4 4 +#define CDMA_ATOMIC_LEN_8 8 +#define CDMA_ATOMIC_LEN_16 16 + +#define SQE_PLD_TOKEN_ID_MASK GENMASK(19, 0) + +/* thanks to include/rdma/ib_verbs.h */ +enum cdma_sq_opcode { + CDMA_OPC_WRITE = 0x3, + CDMA_OPC_WRITE_WITH_NOTIFY = 0x5, + CDMA_OPC_READ = 0x6, + CDMA_OPC_CAS, + CDMA_OPC_FAA = 0xb, + CDMA_OPC_INVALID = 0x12, +}; + +enum cdma_jfsc_mode { + CDMA_JFS_MODE, + CDMA_JETTY_MODE, +}; + +enum cdma_jetty_state { + CDMA_JETTY_RESET, + CDMA_JETTY_READY, + CDMA_JETTY_SUSPENDED, + CDMA_JETTY_ERROR, +}; + +enum cdma_jetty_type { + CDMA_JETTY_ROL = 2, + CDMA_JETTY_ROI, + CDMA_JETTY_TYPE_RESERVED, +}; + +struct cdma_jetty_queue { + struct cdma_buf buf; + void *kva_curr; + u32 id; + void __iomem *db_addr; + void __iomem *dwqe_addr; + u32 pi; + u32 ci; + spinlock_t lock; + u32 max_inline_size; + u32 max_sge_num; + u32 tid; + bool flush_flag; + bool is_jetty; + u32 sqe_bb_cnt; + enum cdma_jetty_state state; + u32 non_pin; + u32 ta_tmo; +}; + +struct cdma_umem_param { + struct cdma_dev *dev; + u64 va; + u64 len; + union cdma_umem_flag flag; + bool is_kernel; +}; + +static inline u64 cdma_cal_npages(u64 va, u64 len) +{ + return (ALIGN(va + len, PAGE_SIZE) - ALIGN_DOWN(va, PAGE_SIZE)) / + PAGE_SIZE; +} + +struct cdma_umem *cdma_umem_get(struct cdma_dev *cdev, u64 va, u64 len, + bool is_kernel); +void cdma_umem_release(struct cdma_umem *umem, bool is_kernel); + +int cdma_k_alloc_buf(struct cdma_dev *cdev, size_t memory_size, + struct cdma_buf *buf); +void cdma_k_free_buf(struct cdma_dev *cdev, size_t memory_size, + struct cdma_buf *buf); +int cdma_pin_queue_addr(struct cdma_dev *cdev, u64 addr, u32 len, + struct cdma_buf *buf); +void cdma_unpin_queue_addr(struct cdma_umem *umem); + +#endif diff --git a/drivers/ub/cdma/cdma_context.c b/drivers/ub/cdma/cdma_context.c new file mode 100644 index 0000000000000000000000000000000000000000..c95ccb0c28b476030a2b5e92e129c5f73e75de9f --- /dev/null +++ b/drivers/ub/cdma/cdma_context.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "CDMA: " fmt + +#include +#include +#include "cdma.h" +#include "cdma_queue.h" +#include "cdma_jfc.h" +#include "cdma_jfs.h" +#include "cdma_tp.h" +#include "cdma_segment.h" +#include "cdma_context.h" + +static void cdma_ctx_handle_free(struct cdma_dev *cdev, + struct cdma_context *ctx) +{ + spin_lock(&cdev->ctx_lock); + idr_remove(&cdev->ctx_idr, ctx->handle); + spin_unlock(&cdev->ctx_lock); +} + +static int cdma_ctx_handle_alloc(struct cdma_dev *cdev, + struct cdma_context *ctx) +{ +#define CDMA_CTX_START 0 +#define CDMA_CTX_END 0xff + int id; + + idr_preload(GFP_KERNEL); + spin_lock(&cdev->ctx_lock); + id = idr_alloc(&cdev->ctx_idr, ctx, CDMA_CTX_START, CDMA_CTX_END, + GFP_NOWAIT); + spin_unlock(&cdev->ctx_lock); + idr_preload_end(); + + return id; +} + +struct cdma_context *cdma_find_ctx_by_handle(struct cdma_dev *cdev, int handle) +{ + struct cdma_context *ctx; + + spin_lock(&cdev->ctx_lock); + ctx = idr_find(&cdev->ctx_idr, handle); + spin_unlock(&cdev->ctx_lock); + + return ctx; +} + +static int cdma_ctx_alloc_tid(struct cdma_dev *cdev, struct cdma_context *ctx) +{ + struct ummu_param drvdata = { .mode = MAPT_MODE_TABLE }; + int ret; + + if (ctx->is_kernel) + ctx->sva = ummu_ksva_bind_device(cdev->dev, &drvdata); + else + ctx->sva = ummu_sva_bind_device(cdev->dev, current->mm, NULL); + + if (!ctx->sva) { + dev_err(cdev->dev, "%s bind device failed.\n", + ctx->is_kernel ? "KSVA" : "SVA"); + return -EFAULT; + } + + ret = ummu_get_tid(cdev->dev, ctx->sva, &ctx->tid); + if (ret) { + dev_err(cdev->dev, "get tid failed, ret = %d.\n", ret); + if (ctx->is_kernel) + ummu_ksva_unbind_device(ctx->sva); + else + ummu_sva_unbind_device(ctx->sva); + } + + return ret; +} + +static void cdma_ctx_free_tid(struct cdma_dev *cdev, struct cdma_context *ctx) +{ + if (ctx->is_kernel) + ummu_ksva_unbind_device(ctx->sva); + else + ummu_sva_unbind_device(ctx->sva); +} + +struct cdma_context *cdma_alloc_context(struct cdma_dev *cdev, bool is_kernel) +{ + struct cdma_context *ctx; + int ret; + + if (!cdev) + return ERR_PTR(-EINVAL); + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return ERR_PTR(-ENOMEM); + + ctx->handle = cdma_ctx_handle_alloc(cdev, ctx); + if (ctx->handle < 0) { + dev_err(cdev->dev, + "Alloc context handle failed, ret = %d.\n", ctx->handle); + ret = ctx->handle; + goto free_ctx; + } + + ctx->cdev = cdev; + ctx->is_kernel = is_kernel; + ret = cdma_ctx_alloc_tid(cdev, ctx); + if (ret) { + dev_err(cdev->dev, "alloc ctx tid failed, ret = %d.\n", ret); + goto free_handle; + } + + spin_lock_init(&ctx->lock); + INIT_LIST_HEAD(&ctx->pgdir_list); + mutex_init(&ctx->pgdir_mutex); + INIT_LIST_HEAD(&ctx->queue_list); + INIT_LIST_HEAD(&ctx->seg_list); + + return ctx; + +free_handle: + cdma_ctx_handle_free(cdev, ctx); +free_ctx: + kfree(ctx); + return ERR_PTR(ret); +} + +void cdma_free_context(struct cdma_dev *cdev, struct cdma_context *ctx) +{ + if (!cdev || !ctx) + return; + + cdma_ctx_free_tid(cdev, ctx); + cdma_ctx_handle_free(cdev, ctx); + mutex_destroy(&ctx->pgdir_mutex); + kfree(ctx); +} + +static void cdma_cleanup_queue_res(struct cdma_dev *cdev, struct cdma_context *ctx) +{ + struct cdma_table *queue_tbl = &cdev->queue_table; + struct cdma_queue *queue, *next_queue; + + list_for_each_entry_safe(queue, next_queue, &ctx->queue_list, list) { + list_del(&queue->list); + + if (queue->jfs) + cdma_delete_jfs(cdev, queue->jfs->id); + + if (queue->tp) + cdma_delete_ctp(cdev, queue->tp->tp_id); + + if (queue->jfc) + cdma_delete_jfc(cdev, queue->jfc->id, NULL); + + spin_lock(&queue_tbl->lock); + idr_remove(&queue_tbl->idr_tbl.idr, queue->id); + spin_unlock(&queue_tbl->lock); + kfree(queue); + } +} + +static void cdma_cleanup_segment_res(struct cdma_dev *cdev, struct cdma_context *ctx) +{ + struct cdma_segment *segment, *next_segment; + + list_for_each_entry_safe(segment, next_segment, &ctx->seg_list, list) { + list_del(&segment->list); + cdma_unregister_seg(cdev, segment); + } +} + +void cdma_cleanup_context_res(struct cdma_context *ctx) +{ + cdma_cleanup_queue_res(ctx->cdev, ctx); + cdma_cleanup_segment_res(ctx->cdev, ctx); + cdma_free_context(ctx->cdev, ctx); +} diff --git a/drivers/ub/cdma/cdma_context.h b/drivers/ub/cdma/cdma_context.h new file mode 100644 index 0000000000000000000000000000000000000000..0eb40763c29deddcaedd6d79a4bdd354981547aa --- /dev/null +++ b/drivers/ub/cdma/cdma_context.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_CONTEXT_H__ +#define __CDMA_CONTEXT_H__ + +#include +#include +#include +#include +#include + +struct cdma_context { + struct dma_context base_ctx; + struct cdma_dev *cdev; + struct iommu_sva *sva; + struct list_head pgdir_list; + struct mutex pgdir_mutex; + spinlock_t lock; + int handle; + u32 tid; + void *jfae; + bool is_kernel; + atomic_t ref_cnt; + struct list_head queue_list; + struct list_head seg_list; + bool invalid; +}; + +struct cdma_ctx_res { + struct cdma_context *ctx; + struct xarray queue_xa; + struct xarray seg_xa; +}; + +struct cdma_context *cdma_find_ctx_by_handle(struct cdma_dev *cdev, int handle); +struct cdma_context *cdma_alloc_context(struct cdma_dev *cdev, bool is_kernel); +void cdma_free_context(struct cdma_dev *cdev, struct cdma_context *ctx); +void cdma_cleanup_context_res(struct cdma_context *ctx); + +#endif /* CDMA_CONTEXT_H */ diff --git a/drivers/ub/cdma/cdma_db.c b/drivers/ub/cdma/cdma_db.c new file mode 100644 index 0000000000000000000000000000000000000000..d241144a684596ea976e90d34f9e5f8e4da9e654 --- /dev/null +++ b/drivers/ub/cdma/cdma_db.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "CDMA: " fmt + +#include +#include "cdma_common.h" +#include "cdma_context.h" +#include "cdma_db.h" + +static int cdma_alloc_db_from_page(struct cdma_k_sw_db_page *page, + struct cdma_sw_db *db) +{ + u32 index; + + index = find_first_bit(page->bitmap, page->num_db); + if (index == page->num_db) + return -ENOMEM; + + clear_bit(index, page->bitmap); + + db->index = index; + db->kpage = page; + db->db_addr = page->db_buf.addr + db->index * CDMA_DB_SIZE; + db->db_record = (u32 *)(page->db_buf.kva + db->index * CDMA_DB_SIZE); + + return 0; +} + +static struct cdma_k_sw_db_page *cdma_alloc_db_page(struct cdma_dev *dev) +{ + struct cdma_k_sw_db_page *page; + int ret; + + page = kzalloc(sizeof(*page), GFP_KERNEL); + if (!page) + return NULL; + + page->num_db = PAGE_SIZE / CDMA_DB_SIZE; + + page->bitmap = bitmap_alloc(page->num_db, GFP_KERNEL); + if (!page->bitmap) { + dev_err(dev->dev, "alloc db bitmap failed.\n"); + goto err_bitmap; + } + + bitmap_fill(page->bitmap, page->num_db); + + ret = cdma_k_alloc_buf(dev, PAGE_SIZE, &page->db_buf); + if (ret) + goto err_kva; + + return page; +err_kva: + bitmap_free(page->bitmap); +err_bitmap: + kfree(page); + + return NULL; +} + +static void cdma_free_db_page(struct cdma_dev *cdev, struct cdma_sw_db *db) +{ + cdma_k_free_buf(cdev, PAGE_SIZE, &db->kpage->db_buf); + bitmap_free(db->kpage->bitmap); + kfree(db->kpage); + db->kpage = NULL; +} + +int cdma_pin_sw_db(struct cdma_context *ctx, struct cdma_sw_db *db) +{ + u64 page_addr = db->db_addr & PAGE_MASK; + struct cdma_sw_db_page *page; + int ret = 0; + + mutex_lock(&ctx->pgdir_mutex); + + list_for_each_entry(page, &ctx->pgdir_list, list) { + if (page->user_virt == page_addr) + goto found; + } + + page = kzalloc(sizeof(*page), GFP_KERNEL); + if (!page) { + ret = -ENOMEM; + goto out; + } + + refcount_set(&page->refcount, 1); + page->user_virt = page_addr; + page->umem = cdma_umem_get(ctx->cdev, page_addr, PAGE_SIZE, false); + if (IS_ERR(page->umem)) { + ret = PTR_ERR(page->umem); + dev_err(ctx->cdev->dev, "get umem failed, ret = %d.\n", ret); + kfree(page); + goto out; + } + + list_add(&page->list, &ctx->pgdir_list); + db->page = page; + mutex_unlock(&ctx->pgdir_mutex); + return 0; + +found: + db->page = page; + refcount_inc(&page->refcount); +out: + mutex_unlock(&ctx->pgdir_mutex); + + return ret; +} + +void cdma_unpin_sw_db(struct cdma_context *ctx, struct cdma_sw_db *db) +{ + mutex_lock(&ctx->pgdir_mutex); + + if (refcount_dec_and_test(&db->page->refcount)) { + list_del(&db->page->list); + cdma_umem_release(db->page->umem, false); + kfree(db->page); + db->page = NULL; + } + + mutex_unlock(&ctx->pgdir_mutex); +} + +int cdma_alloc_sw_db(struct cdma_dev *cdev, struct cdma_sw_db *db) +{ + struct cdma_k_sw_db_page *page; + int ret = 0; + + mutex_lock(&cdev->db_mutex); + + list_for_each_entry(page, &cdev->db_page, list) + if (!cdma_alloc_db_from_page(page, db)) + goto out; + + page = cdma_alloc_db_page(cdev); + if (!page) { + ret = -ENOMEM; + dev_err(cdev->dev, "alloc sw db page failed.\n"); + goto out; + } + + list_add(&page->list, &cdev->db_page); + + ret = cdma_alloc_db_from_page(page, db); + if (ret) + dev_err(cdev->dev, "alloc sw db from page failed, ret = %d.\n", ret); +out: + mutex_unlock(&cdev->db_mutex); + + return ret; +} + +void cdma_free_sw_db(struct cdma_dev *cdev, struct cdma_sw_db *db) +{ + mutex_lock(&cdev->db_mutex); + + set_bit(db->index, db->kpage->bitmap); + + if (bitmap_full(db->kpage->bitmap, db->kpage->num_db)) { + list_del(&db->kpage->list); + cdma_free_db_page(cdev, db); + } + + mutex_unlock(&cdev->db_mutex); +} diff --git a/drivers/ub/cdma/cdma_db.h b/drivers/ub/cdma/cdma_db.h new file mode 100644 index 0000000000000000000000000000000000000000..fa3ef8c0f5703af1a5b06de171338bdb0f2ed054 --- /dev/null +++ b/drivers/ub/cdma/cdma_db.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_DB_H__ +#define __CDMA_DB_H__ + +struct cdma_context; +struct cdma_dev; + +struct cdma_sw_db_page { + struct list_head list; + struct cdma_umem *umem; + u64 user_virt; + refcount_t refcount; +}; + +struct cdma_k_sw_db_page { + struct list_head list; + u32 num_db; + unsigned long *bitmap; + struct cdma_buf db_buf; +}; + +struct cdma_sw_db { + union { + struct cdma_sw_db_page *page; + struct cdma_k_sw_db_page *kpage; + }; + u32 index; + u64 db_addr; + u32 *db_record; +}; + +int cdma_pin_sw_db(struct cdma_context *ctx, struct cdma_sw_db *db); + +void cdma_unpin_sw_db(struct cdma_context *ctx, struct cdma_sw_db *db); + +int cdma_alloc_sw_db(struct cdma_dev *dev, struct cdma_sw_db *db); + +void cdma_free_sw_db(struct cdma_dev *dev, struct cdma_sw_db *db); + +#endif /* CDMA_DB_H */ diff --git a/drivers/ub/cdma/cdma_debugfs.c b/drivers/ub/cdma/cdma_debugfs.c new file mode 100644 index 0000000000000000000000000000000000000000..d0de451e92caae4706a01a836fb9d18524d58ae0 --- /dev/null +++ b/drivers/ub/cdma/cdma_debugfs.c @@ -0,0 +1,783 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "CDMA: " fmt + +#include +#include +#include +#include +#include +#include "cdma_queue.h" +#include "cdma.h" +#include "cdma_jfc.h" +#include "cdma_jfs.h" +#include "cdma_mbox.h" +#include "cdma_cmd.h" +#include "cdma_debugfs.h" + +#define CDMA_DBG_READ_LEN 65536 +#define BUF_10_BASE 10 +#define BUF_SIZE 8 + +/* ctx debugfs start */ +static void cdma_get_ctx_info(struct cdma_dev *cdev, + struct cdma_queue *queue, + enum cdma_dbg_ctx_type ctx_type, + struct cdma_ctx_info *ctx_info) +{ + struct auxiliary_device *adev = cdev->adev; + +#define CDMA_DBG_CTX_SIZE_256 256 +#define UBASE_CTX_SIZE_128 128 + switch (ctx_type) { + case CDMA_DBG_JFS_CTX: + ctx_info->start_idx = queue->jfs_id; + ctx_info->ctx_size = CDMA_DBG_CTX_SIZE_256; + ctx_info->op = UBASE_MB_QUERY_JFS_CONTEXT; + ctx_info->ctx_name = "jfs"; + break; + case CDMA_DBG_SQ_JFC_CTX: + ctx_info->start_idx = queue->jfc_id; + ctx_info->ctx_size = UBASE_CTX_SIZE_128; + ctx_info->op = UBASE_MB_QUERY_JFC_CONTEXT; + ctx_info->ctx_name = "sq_jfc"; + break; + default: + dev_err(&adev->dev, "get ctx info failed, ctx_type = %d.\n", + ctx_type); + break; + } +} + +static void cdma_print_ctx_hw_bytype(struct seq_file *s, + enum cdma_dbg_ctx_type ctx_type, + struct cdma_ctx_info *ctx_info, + struct ubase_cmd_mailbox *mailbox) +{ + struct cdma_jfs_ctx *jfs_ctx; + struct cdma_jfc_ctx *jfc_ctx; + + seq_printf(s, "offset\t%s%u\n", ctx_info->ctx_name, ctx_info->start_idx); + + if (ctx_type == CDMA_DBG_JFS_CTX) { + jfs_ctx = (struct cdma_jfs_ctx *)mailbox->buf; + jfs_ctx->sqe_base_addr_l = 0; + jfs_ctx->sqe_base_addr_h = 0; + jfs_ctx->user_data_l = 0; + jfs_ctx->user_data_h = 0; + ubase_print_context_hw(s, jfs_ctx, ctx_info->ctx_size); + } else if (ctx_type == CDMA_DBG_SQ_JFC_CTX) { + jfc_ctx = (struct cdma_jfc_ctx *)mailbox->buf; + jfc_ctx->cqe_va_l = 0; + jfc_ctx->cqe_va_h = 0; + jfc_ctx->cqe_token_value = 0; + jfc_ctx->record_db_addr_l = 0; + jfc_ctx->record_db_addr_h = 0; + jfc_ctx->remote_token_value = 0; + ubase_print_context_hw(s, jfc_ctx, ctx_info->ctx_size); + } + + seq_puts(s, "\n"); +} + +static int cdma_dbg_dump_ctx_hw(struct seq_file *s, enum cdma_dbg_ctx_type ctx_type) +{ + struct cdma_dev *cdev = dev_get_drvdata(s->private); + struct auxiliary_device *adev = cdev->adev; + u32 queue_id = cdev->cdbgfs.cfg.queue_id; + struct cdma_ctx_info ctx_info = { 0 }; + struct ubase_cmd_mailbox *mailbox; + struct ubase_mbx_attr attr; + struct cdma_queue *queue; + + spin_lock(&cdev->queue_table.lock); + queue = idr_find(&cdev->queue_table.idr_tbl.idr, queue_id); + if (!queue) { + spin_unlock(&cdev->queue_table.lock); + dev_err(&adev->dev, "find queue[%u] for dump context hw failed.\n", queue_id); + return -EINVAL; + } + + if (!queue->jfs_id) { + spin_unlock(&cdev->queue_table.lock); + dev_warn(&adev->dev, "queue resource is not initialized.\n"); + return -EINVAL; + } + + cdma_get_ctx_info(cdev, queue, ctx_type, &ctx_info); + spin_unlock(&cdev->queue_table.lock); + + cdma_fill_mbx_attr(&attr, ctx_info.start_idx, ctx_info.op, 0); + mailbox = cdma_mailbox_query_ctx(cdev, &attr); + if (!mailbox) { + dev_err(&adev->dev, "cdma dbg post query %s ctx mbx failed.\n", + ctx_info.ctx_name); + return -ENOMEM; + } + + cdma_print_ctx_hw_bytype(s, ctx_type, &ctx_info, mailbox); + + cdma_free_cmd_mailbox(cdev, mailbox); + + return 0; +} + +static int cdma_dbg_dump_jfs_ctx_hw(struct seq_file *s, void *data) +{ + if (!s || !s->private) + return -EINVAL; + + return cdma_dbg_dump_ctx_hw(s, CDMA_DBG_JFS_CTX); +} + +static int cdma_dbg_dump_sq_jfc_ctx_hw(struct seq_file *s, void *data) +{ + if (!s || !s->private) + return -EINVAL; + + return cdma_dbg_dump_ctx_hw(s, CDMA_DBG_SQ_JFC_CTX); +} + +static void cdma_get_jfs_cfg(struct cdma_queue *queue, struct seq_file *s) +{ + struct cdma_jfs_cfg *cfg; + + if (!queue->jfs) + return; + + cfg = &queue->jfs->cfg; + seq_printf(s, "%-13u", cfg->depth); + seq_printf(s, "%-12u", cfg->flag.value); + seq_printf(s, "%-17u", cfg->eid_index); + seq_printf(s, "%-10u", cfg->priority); + seq_printf(s, "%-9u", cfg->max_sge); + seq_printf(s, "%-10u", cfg->max_rsge); + seq_printf(s, "%-11u", cfg->rnr_retry); + seq_printf(s, "%-13u", cfg->err_timeout); + seq_printf(s, "%-14u", cfg->jfc_id); + seq_printf(s, "%-15u", cfg->sqe_pos); + seq_printf(s, "%-11u", cfg->tpn); + seq_printf(s, "%-15u", cfg->pld_pos); + seq_printf(s, "%-16u", cfg->queue_id); +} + +static void cdma_get_jfc_cfg(struct cdma_queue *queue, struct seq_file *s) +{ + struct cdma_jfc_cfg *cfg; + + if (!queue->jfc) + return; + + cfg = &queue->jfc->jfc_cfg; + seq_printf(s, "%-13u", cfg->depth); + seq_printf(s, "%-12u", cfg->ceqn); + seq_printf(s, "%-16u", cfg->queue_id); +} + +static void cdma_get_jfs_title(struct seq_file *s) +{ + seq_puts(s, "depth "); + seq_puts(s, "flag "); + seq_puts(s, "eid_index "); + seq_puts(s, "priority "); + seq_puts(s, "max_sge "); + seq_puts(s, "max_rsge "); + seq_puts(s, "rnr_retry "); + seq_puts(s, "err_timeout "); + seq_puts(s, "jfc_id "); + seq_puts(s, "sqe_pos "); + seq_puts(s, "tpn "); + seq_puts(s, "pld_pos "); + seq_puts(s, "queue_id "); + seq_puts(s, "\n"); +} + +static void cdma_get_jfc_title(struct seq_file *s) +{ + seq_puts(s, "depth "); + seq_puts(s, "flag "); + seq_puts(s, "ceqn "); + seq_puts(s, "queue_id "); + seq_puts(s, "\n"); +} + +static int cdma_dbg_dump_ctx(struct seq_file *s, enum cdma_dbg_ctx_type ctx_type) +{ + struct cdma_dbg_context { + void (*get_title)(struct seq_file *s); + void (*get_cfg)(struct cdma_queue *queue, struct seq_file *s); + } dbg_ctx[] = { + {cdma_get_jfs_title, cdma_get_jfs_cfg}, + {cdma_get_jfc_title, cdma_get_jfc_cfg}, + }; + struct cdma_dev *cdev = dev_get_drvdata(s->private); + u32 queue_id = cdev->cdbgfs.cfg.queue_id; + struct cdma_queue *queue; + + dbg_ctx[ctx_type].get_title(s); + + spin_lock(&cdev->queue_table.lock); + queue = idr_find(&cdev->queue_table.idr_tbl.idr, queue_id); + if (!queue) { + spin_unlock(&cdev->queue_table.lock); + dev_err(&cdev->adev->dev, "find queue[%u] for dump context failed.\n", queue_id); + return -EINVAL; + } + + dbg_ctx[ctx_type].get_cfg(queue, s); + + spin_unlock(&cdev->queue_table.lock); + + return 0; +} + +int cdma_dbg_dump_jfs_ctx(struct seq_file *s, void *data) +{ + if (!s || !s->private) + return -EINVAL; + + return cdma_dbg_dump_ctx(s, CDMA_DBG_JFS_CTX); +} + +int cdma_dbg_dump_sq_jfc_ctx(struct seq_file *s, void *data) +{ + if (!s || !s->private) + return -EINVAL; + + return cdma_dbg_dump_ctx(s, CDMA_DBG_SQ_JFC_CTX); +} +/* ctx debugfs end */ + +/* resource debugfs start */ +static int cdma_dbg_dump_dev_info(struct seq_file *s, void *data) +{ + if (!s || !s->private) + return -EINVAL; + + struct cdma_dev *cdev = dev_get_drvdata(s->private); + u8 eu_num = cdev->base.attr.eu_num; + u32 seid_idx, seid, upi, i; + + seq_printf(s, "EU_ENTRY_NUM: %u\n", eu_num); + for (i = 0; i < eu_num; i++) { + seid_idx = cdev->base.attr.eus[i].eid_idx; + seid = cdev->base.attr.eus[i].eid.dw0; + upi = cdev->base.attr.eus[i].upi; + seq_printf(s, "SEID_IDX: %u, SEID: %u, UPI: %u\n", seid_idx, seid, upi); + } + + return 0; +} + +static int cdma_dbg_dump_cap_info(struct seq_file *s, void *data) +{ + if (!s || !s->private) + return -EINVAL; + + struct cdma_dev *cdev = dev_get_drvdata(s->private); + struct cdma_caps *caps = &cdev->caps; + + seq_printf(s, "MAX_JFC: %u\n", caps->jfc.max_cnt); + seq_printf(s, "MAX_JFS: %u\n", caps->jfs.max_cnt); + seq_printf(s, "MAX_JFC_DEPTH: %u\n", caps->jfc.depth); + seq_printf(s, "MAX_JFS_DEPTH: %u\n", caps->jfs.depth); + seq_printf(s, "MAX_JFS_SGE: %u\n", caps->jfs_sge); + seq_printf(s, "MAX_JFS_RSGE: %u\n", caps->jfs_rsge); + seq_printf(s, "MAX_MSG_SIZE: %u\n", caps->max_msg_len); + seq_printf(s, "TRANS_MODE: %u\n", caps->trans_mode); + seq_printf(s, "CEQ_CNT: %u\n", caps->comp_vector_cnt); + + return 0; +} + +static int cdma_dbg_dump_queue_info(struct seq_file *s, void *data) +{ + if (!s || !s->private) + return -EINVAL; + + struct cdma_dev *cdev = dev_get_drvdata(s->private); + u32 queue_id = cdev->cdbgfs.cfg.queue_id; + struct cdma_queue *queue; + + spin_lock(&cdev->queue_table.lock); + queue = idr_find(&cdev->queue_table.idr_tbl.idr, queue_id); + if (!queue) { + spin_unlock(&cdev->queue_table.lock); + dev_err(&cdev->adev->dev, "find queue[%u] for dump queue info failed.\n", queue_id); + return -EINVAL; + } + + seq_printf(s, "QUEUE_DEPTH: %u\n", queue->cfg.queue_depth); + seq_printf(s, "DST CNA: 0x%x\n", queue->cfg.dcna); + seq_printf(s, "RMT EID: 0x%x\n", queue->cfg.rmt_eid.dw0); + seq_printf(s, "PRIORITY: %u\n", queue->cfg.priority); + + spin_unlock(&cdev->queue_table.lock); + + return 0; +} +/* resource debugfs end */ + +/* entry info start */ +static void cdma_dbg_dump_sqe_info(struct cdma_sqe_ctl *sqe_ctl, struct seq_file *s) +{ + seq_printf(s, "sqe bb idx: %u\n", sqe_ctl->sqe_bb_idx); + seq_printf(s, "place odr: %u\n", sqe_ctl->place_odr); + seq_printf(s, "comp order: %u\n", sqe_ctl->comp_order); + seq_printf(s, "fence: %u\n", sqe_ctl->fence); + seq_printf(s, "se: %u\n", sqe_ctl->se); + seq_printf(s, "cqe: %u\n", sqe_ctl->cqe); + seq_printf(s, "owner: %u\n", sqe_ctl->owner); + seq_printf(s, "opcode: %u\n", sqe_ctl->opcode); + seq_printf(s, "tpn: %u\n", sqe_ctl->tpn); + seq_printf(s, "sge num: %u\n", sqe_ctl->sge_num); + seq_printf(s, "rmt eid: %u\n", sqe_ctl->rmt_eid[0]); +} + +static void cdma_dbg_dump_cqe_info(struct cdma_jfc_cqe *cqe, struct seq_file *s) +{ + seq_printf(s, "sr: %u\n", cqe->s_r); + seq_printf(s, "owner: %u\n", cqe->owner); + seq_printf(s, "opcode: %u\n", cqe->opcode); + seq_printf(s, "fd: %u\n", cqe->fd); + seq_printf(s, "substatus: %u\n", cqe->substatus); + seq_printf(s, "status: %u\n", cqe->status); + seq_printf(s, "entry idx: %u\n", cqe->entry_idx); + seq_printf(s, "tpn: %u\n", cqe->tpn); + seq_printf(s, "rmt eid: %u\n", cqe->rmt_eid[0]); + seq_printf(s, "byte cnt: %u\n", cqe->byte_cnt); +} + +static void cdma_dbg_dum_eu(struct cdma_dev *cdev, int i, struct seq_file *s) +{ + struct eu_info *eu = &cdev->base.attr.eus[i]; + + seq_printf(s, "%d: ", i); + seq_printf(s, "idx[0x%x] ", eu->eid_idx); + seq_printf(s, "eid[0x%x] ", eu->eid.dw0); + seq_printf(s, "upi[0x%x]\n", eu->upi); +} + +static int cdma_dbg_dump_sqe(struct seq_file *s, void *data) +{ + if (!s || !s->private) + return -EINVAL; + + struct cdma_dev *cdev = dev_get_drvdata(s->private); + u32 queue_id = cdev->cdbgfs.cfg.queue_id; + u32 entry_pi = cdev->cdbgfs.cfg.entry_pi; + struct cdma_sqe_ctl *sqe_ctl; + struct cdma_queue *queue; + struct cdma_jfs *jfs; + + spin_lock(&cdev->queue_table.lock); + queue = idr_find(&cdev->queue_table.idr_tbl.idr, queue_id); + if (!queue) { + spin_unlock(&cdev->queue_table.lock); + dev_err(&cdev->adev->dev, "find queue[%u] for dump sqe failed.\n", queue_id); + return -EINVAL; + } + + if (queue->jfs && queue->is_kernel) { + jfs = to_cdma_jfs(queue->jfs); + if (entry_pi >= jfs->base_jfs.cfg.depth) { + spin_unlock(&cdev->queue_table.lock); + dev_err(&cdev->adev->dev, "pi [%u] overflow for dump sqe.\n", entry_pi); + return -EINVAL; + } + + spin_lock(&jfs->sq.lock); + sqe_ctl = (struct cdma_sqe_ctl *)(jfs->sq.buf.kva + + (entry_pi & (jfs->sq.buf.entry_cnt - 1)) * + jfs->sq.buf.entry_size); + cdma_dbg_dump_sqe_info(sqe_ctl, s); + spin_unlock(&jfs->sq.lock); + } else { + dev_warn(&cdev->adev->dev, "not support queue[%u] for dump sqe.\n", queue_id); + } + + spin_unlock(&cdev->queue_table.lock); + + return 0; +} + +static int cdma_dbg_dump_cqe(struct seq_file *s, void *data) +{ + if (!s || !s->private) + return -EINVAL; + + struct cdma_dev *cdev = dev_get_drvdata(s->private); + u32 queue_id = cdev->cdbgfs.cfg.queue_id; + u32 entry_ci = cdev->cdbgfs.cfg.entry_ci; + struct cdma_queue *queue; + struct cdma_jfc_cqe *cqe; + struct cdma_jfc *jfc; + + spin_lock(&cdev->queue_table.lock); + queue = idr_find(&cdev->queue_table.idr_tbl.idr, queue_id); + if (!queue) { + spin_unlock(&cdev->queue_table.lock); + dev_err(&cdev->adev->dev, "find queue[%u] for dump cqe failed.\n", queue_id); + return -EINVAL; + } + + if (queue->jfc && queue->is_kernel) { + jfc = to_cdma_jfc(queue->jfc); + if (entry_ci >= jfc->base.jfc_cfg.depth) { + spin_unlock(&cdev->queue_table.lock); + dev_err(&cdev->adev->dev, "ci [%u] overflow for dump cqe.\n", entry_ci); + return -EINVAL; + } + + spin_lock(&jfc->lock); + cqe = (struct cdma_jfc_cqe *)(jfc->buf.kva + + (entry_ci & (jfc->buf.entry_cnt - 1)) * + jfc->buf.entry_size); + cdma_dbg_dump_cqe_info(cqe, s); + spin_unlock(&jfc->lock); + } else { + dev_warn(&cdev->adev->dev, "not support queue[%u] for dump cqe.\n", queue_id); + } + + spin_unlock(&cdev->queue_table.lock); + + return 0; +} + +/* Dump eu info */ +static int cdma_dbg_dump_eu(struct seq_file *s, void *data) +{ + if (!s || !s->private) + return -EINVAL; + + struct cdma_dev *cdev = dev_get_drvdata(s->private); + int ret, i; + + ret = cdma_ctrlq_query_eu(cdev); + if (ret) + return ret; + + for (i = 0; i < cdev->base.attr.eu_num; i++) + cdma_dbg_dum_eu(cdev, i, s); + + return 0; +} +/* entry info end */ + +static bool cdma_dbg_dentry_support(struct device *dev, u32 property) +{ + struct cdma_dev *cdev = dev_get_drvdata(dev); + + return ubase_dbg_dentry_support(cdev->adev, property); +} + +static struct ubase_dbg_dentry_info cdma_dbg_dentry[] = { + { + .name = "context", + .property = UBASE_SUP_CDMA | UBASE_SUP_UBL, + .support = cdma_dbg_dentry_support, + }, { + .name = "resource_info", + .property = UBASE_SUP_CDMA | UBASE_SUP_UBL, + .support = cdma_dbg_dentry_support, + }, { + .name = "entry_info", + .property = UBASE_SUP_CDMA | UBASE_SUP_UBL, + .support = cdma_dbg_dentry_support, + }, + /* keep "cdma" at the bottom and add new directory above */ + { + .name = "cdma", + .property = UBASE_SUP_CDMA | UBASE_SUP_UBL, + .support = cdma_dbg_dentry_support, + }, +}; + +static struct ubase_dbg_cmd_info cdma_dbg_cmd[] = { + { + .name = "jfs_context", + .dentry_index = CDMA_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_CDMA | UBASE_SUP_UBL, + .support = cdma_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = cdma_dbg_dump_jfs_ctx, + }, { + .name = "sq_jfc_context", + .dentry_index = CDMA_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_CDMA | UBASE_SUP_UBL, + .support = cdma_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = cdma_dbg_dump_sq_jfc_ctx, + }, { + .name = "jfs_context_hw", + .dentry_index = CDMA_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_CDMA | UBASE_SUP_UBL, + .support = cdma_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = cdma_dbg_dump_jfs_ctx_hw, + }, { + .name = "sq_jfc_context_hw", + .dentry_index = CDMA_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_CDMA | UBASE_SUP_UBL, + .support = cdma_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = cdma_dbg_dump_sq_jfc_ctx_hw, + }, { + .name = "dev_info", + .dentry_index = CDMA_DBG_DENTRY_RES_INFO, + .property = UBASE_SUP_CDMA | UBASE_SUP_UBL, + .support = cdma_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = cdma_dbg_dump_dev_info, + }, { + .name = "cap_info", + .dentry_index = CDMA_DBG_DENTRY_RES_INFO, + .property = UBASE_SUP_CDMA | UBASE_SUP_UBL, + .support = cdma_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = cdma_dbg_dump_cap_info, + }, { + .name = "queue_info", + .dentry_index = CDMA_DBG_DENTRY_RES_INFO, + .property = UBASE_SUP_CDMA | UBASE_SUP_UBL, + .support = cdma_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = cdma_dbg_dump_queue_info, + }, { + .name = "sqe", + .dentry_index = CDMA_DBG_DENTRY_ENTRY_INFO, + .property = UBASE_SUP_CDMA | UBASE_SUP_UBL, + .support = cdma_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = cdma_dbg_dump_sqe, + }, { + .name = "cqe", + .dentry_index = CDMA_DBG_DENTRY_ENTRY_INFO, + .property = UBASE_SUP_CDMA | UBASE_SUP_UBL, + .support = cdma_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = cdma_dbg_dump_cqe, + }, { + .name = "eu", + .dentry_index = CDMA_DBG_DENTRY_ENTRY_INFO, + .property = UBASE_SUP_CDMA | UBASE_SUP_UBL, + .support = cdma_dbg_dentry_support, + .init = ubase_dbg_seq_file_init, + .read_func = cdma_dbg_dump_eu, + }, +}; + +static ssize_t cdma_dbgfs_cfg_write_val(struct file *filp, + const char __user *buffer, + size_t count, loff_t *ppos, + enum cdma_dbgfs_cfg_type type) +{ + struct cdma_dbgfs_cfg *cfg = (struct cdma_dbgfs_cfg *)filp->private_data; + char buf[BUF_SIZE] = { 0 }; + ssize_t len, ret; + u32 value; + + len = simple_write_to_buffer(buf, BUF_SIZE - 1, ppos, buffer, count); + if (len < 0) + return len; + + ret = kstrtouint(buf, BUF_10_BASE, &value); + if (ret) + return ret; + + switch (type) { + case CDMA_QUEUE_ID: + cfg->queue_id = value; + break; + case CDMA_ENTRY_PI: + cfg->entry_pi = value; + break; + case CDMA_ENTRY_CI: + cfg->entry_ci = value; + break; + default: + return -EINVAL; + } + + return len; +} + +static ssize_t cdma_dbgfs_cfg_read_val(struct file *filp, + char *buffer, size_t count, loff_t *ppos, + enum cdma_dbgfs_cfg_type type) +{ + struct cdma_dbgfs_cfg *cfg = (struct cdma_dbgfs_cfg *)filp->private_data; + char buf[BUF_SIZE] = { 0 }; + u32 value = 0; + size_t len; + + switch (type) { + case CDMA_QUEUE_ID: + value = cfg->queue_id; + break; + case CDMA_ENTRY_PI: + value = cfg->entry_pi; + break; + case CDMA_ENTRY_CI: + value = cfg->entry_ci; + break; + default: + break; + } + + len = scnprintf(buf, sizeof(buf), "%u\n", value); + + return simple_read_from_buffer(buffer, count, ppos, buf, len); +} + +static ssize_t cdma_dbgfs_cfg_write_queue_id(struct file *filp, + const char __user *buffer, + size_t count, loff_t *ppos) +{ + return cdma_dbgfs_cfg_write_val(filp, buffer, count, ppos, CDMA_QUEUE_ID); +} + +static ssize_t cdma_dbgfs_cfg_read_queue_id(struct file *filp, + char *buffer, size_t count, + loff_t *ppos) +{ + return cdma_dbgfs_cfg_read_val(filp, buffer, count, ppos, CDMA_QUEUE_ID); +} + +static ssize_t cdma_dbgfs_cfg_write_entry_pi(struct file *filp, + const char __user *buffer, + size_t count, loff_t *ppos) +{ + return cdma_dbgfs_cfg_write_val(filp, buffer, count, ppos, CDMA_ENTRY_PI); +} + +static ssize_t cdma_dbgfs_cfg_read_entry_pi(struct file *filp, + char *buffer, size_t count, + loff_t *ppos) +{ + return cdma_dbgfs_cfg_read_val(filp, buffer, count, ppos, CDMA_ENTRY_PI); +} + +static ssize_t cdma_dbgfs_cfg_write_entry_ci(struct file *filp, + const char __user *buffer, + size_t count, loff_t *ppos) +{ + return cdma_dbgfs_cfg_write_val(filp, buffer, count, ppos, CDMA_ENTRY_CI); +} + +static ssize_t cdma_dbgfs_cfg_read_entry_ci(struct file *filp, + char *buffer, size_t count, + loff_t *ppos) +{ + return cdma_dbgfs_cfg_read_val(filp, buffer, count, ppos, CDMA_ENTRY_CI); +} + +static struct cdma_dbgfs_cfg_info cdma_dbg_cfg[] = { + { + .name = "queue_id", + {true, true, true}, + {.owner = THIS_MODULE, + .read = cdma_dbgfs_cfg_read_queue_id, + .write = cdma_dbgfs_cfg_write_queue_id, + .open = simple_open, }, + }, { + .name = "entry_pi", + {false, false, true}, + {.owner = THIS_MODULE, + .read = cdma_dbgfs_cfg_read_entry_pi, + .write = cdma_dbgfs_cfg_write_entry_pi, + .open = simple_open, }, + }, { + .name = "entry_ci", + {false, false, true}, + {.owner = THIS_MODULE, + .read = cdma_dbgfs_cfg_read_entry_ci, + .write = cdma_dbgfs_cfg_write_entry_ci, + .open = simple_open, }, + }, +}; + +static int cdma_dbg_create_cfg_file(struct cdma_dev *cdev, + struct ubase_dbg_dentry_info *dentry_info, + u8 array_size) +{ + struct dentry *debugfs_file; + struct dentry *cur_dir; + size_t i, j; + + for (i = 0; i < array_size - 1; i++) { + cur_dir = dentry_info[i].dentry; + for (j = 0; j < ARRAY_SIZE(cdma_dbg_cfg); j++) { + if (!cdma_dbg_cfg[j].dentry_valid[i]) + continue; + debugfs_file = debugfs_create_file(cdma_dbg_cfg[j].name, + 0400, cur_dir, &cdev->cdbgfs.cfg, + &cdma_dbg_cfg[j].file_ops); + if (!debugfs_file) + return -ENOMEM; + } + } + + return 0; +} + +int cdma_dbg_init(struct auxiliary_device *adev) +{ + struct ubase_dbg_dentry_info dbg_dentry[CDMA_DBG_DENTRY_ROOT + 1] = {0}; + struct dentry *ubase_root_dentry = ubase_diag_debugfs_root(adev); + struct device *dev = &adev->dev; + struct cdma_dev *cdev; + int ret; + + cdev = dev_get_drvdata(dev); + + if (!ubase_root_dentry) { + dev_err(dev, "dbgfs root dentry does not exist.\n"); + return -ENOENT; + } + + memcpy(dbg_dentry, cdma_dbg_dentry, sizeof(cdma_dbg_dentry)); + cdev->cdbgfs.dbgfs.dentry = debugfs_create_dir( + dbg_dentry[ARRAY_SIZE(dbg_dentry) - 1].name, ubase_root_dentry); + if (IS_ERR(cdev->cdbgfs.dbgfs.dentry)) { + dev_err(dev, "create cdma debugfs root dir failed.\n"); + return PTR_ERR(cdev->cdbgfs.dbgfs.dentry); + } + + dbg_dentry[CDMA_DBG_DENTRY_ROOT].dentry = cdev->cdbgfs.dbgfs.dentry; + cdev->cdbgfs.dbgfs.cmd_info = cdma_dbg_cmd; + cdev->cdbgfs.dbgfs.cmd_info_size = ARRAY_SIZE(cdma_dbg_cmd); + + ret = ubase_dbg_create_dentry(dev, &cdev->cdbgfs.dbgfs, dbg_dentry, + ARRAY_SIZE(dbg_dentry) - 1); + if (ret) { + dev_err(dev, "create cdma debugfs dentry failed, ret = %d.\n", ret); + goto create_dentry_err; + } + + ret = cdma_dbg_create_cfg_file(cdev, dbg_dentry, ARRAY_SIZE(dbg_dentry)); + if (ret) { + dev_err(dev, "create cdma debugfs cfg file failed, ret = %d.\n", ret); + goto create_dentry_err; + } + + return 0; + +create_dentry_err: + debugfs_remove_recursive(cdev->cdbgfs.dbgfs.dentry); + cdev->cdbgfs.dbgfs.dentry = NULL; + + return ret; +} + +void cdma_dbg_uninit(struct auxiliary_device *adev) +{ + struct cdma_dev *cdev = dev_get_drvdata(&adev->dev); + + if (!cdev->cdbgfs.dbgfs.dentry) + return; + + debugfs_remove_recursive(cdev->cdbgfs.dbgfs.dentry); + cdev->cdbgfs.dbgfs.dentry = NULL; +} diff --git a/drivers/ub/cdma/cdma_debugfs.h b/drivers/ub/cdma/cdma_debugfs.h new file mode 100644 index 0000000000000000000000000000000000000000..1cd0f2ada9dc8b49f0162c627b71f50f9bb2e0d7 --- /dev/null +++ b/drivers/ub/cdma/cdma_debugfs.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_DEBUGFS_H__ +#define __CDMA_DEBUGFS_H__ + +#include +#include + +enum cdma_dbg_dentry_type { + CDMA_DBG_DENTRY_CONTEXT, + CDMA_DBG_DENTRY_RES_INFO, + CDMA_DBG_DENTRY_ENTRY_INFO, + /* must be the last entry. */ + CDMA_DBG_DENTRY_ROOT, +}; + +/* ctx debugfs start */ +struct cdma_ctx_info { + u32 start_idx; + u32 ctx_size; + u8 op; + const char *ctx_name; +}; + +enum cdma_dbg_ctx_type { + CDMA_DBG_JFS_CTX = 0, + CDMA_DBG_SQ_JFC_CTX = 1, +}; +/* ctx debugfs end */ + +struct cdma_dbgfs_cfg_info { + const char *name; + bool dentry_valid[CDMA_DBG_DENTRY_ROOT]; + const struct file_operations file_ops; +}; + +struct cdma_dbgfs_cfg { + u32 queue_id; + u32 entry_pi; + u32 entry_ci; +}; + +enum cdma_dbgfs_cfg_type { + CDMA_QUEUE_ID = 0, + CDMA_ENTRY_PI, + CDMA_ENTRY_CI +}; + +struct cdma_dbgfs { + struct ubase_dbgfs dbgfs; + struct cdma_dbgfs_cfg cfg; +}; + +int cdma_dbg_init(struct auxiliary_device *adev); +void cdma_dbg_uninit(struct auxiliary_device *adev); + +#endif /* CDMA_DEBUGFS_H */ diff --git a/drivers/ub/cdma/cdma_dev.c b/drivers/ub/cdma/cdma_dev.c new file mode 100644 index 0000000000000000000000000000000000000000..2b69a44b346e28bc751782793264e036f5c4f524 --- /dev/null +++ b/drivers/ub/cdma/cdma_dev.c @@ -0,0 +1,466 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "CDMA: " fmt + +#include +#include +#include +#include +#include + +#include "cdma.h" +#include "cdma_cmd.h" +#include "cdma_tid.h" +#include "cdma_context.h" +#include +#include +#include "cdma_common.h" +#include "cdma_tp.h" +#include "cdma_jfs.h" +#include "cdma_jfc.h" +#include "cdma_queue.h" +#include "cdma_dev.h" + +static DEFINE_XARRAY(cdma_devs_tbl); +static atomic_t cdma_devs_num = ATOMIC_INIT(0); + +struct cdma_dev *get_cdma_dev_by_eid(u32 eid) +{ + struct cdma_dev *cdev = NULL; + unsigned long index = 0; + + xa_for_each(&cdma_devs_tbl, index, cdev) + if (cdev->eid == eid) + return cdev; + + return NULL; +} + +struct xarray *get_cdma_dev_tbl(u32 *devs_num) +{ + *devs_num = atomic_read(&cdma_devs_num); + + return &cdma_devs_tbl; +} + +/* Add the device to the device list for user query. */ +static int cdma_add_device_to_list(struct cdma_dev *cdev) +{ + struct auxiliary_device *adev = cdev->adev; + int ret; + + if (adev->id >= CDMA_UE_MAX_NUM) { + dev_err(cdev->dev, "invalid ue id %u.\n", adev->id); + return -EINVAL; + } + + down_write(&g_device_rwsem); + ret = xa_err(xa_store(&cdma_devs_tbl, adev->id, cdev, GFP_KERNEL)); + if (ret) { + dev_err(cdev->dev, + "store cdma device to table failed, adev id = %u.\n", + adev->id); + up_write(&g_device_rwsem); + return ret; + } + + atomic_inc(&cdma_devs_num); + up_write(&g_device_rwsem); + + return 0; +} + +static void cdma_del_device_from_list(struct cdma_dev *cdev) +{ + struct auxiliary_device *adev = cdev->adev; + + if (adev->id >= CDMA_UE_MAX_NUM) { + dev_err(cdev->dev, "invalid ue id %u.\n", adev->id); + return; + } + + down_write(&g_device_rwsem); + atomic_dec(&cdma_devs_num); + xa_erase(&cdma_devs_tbl, adev->id); + up_write(&g_device_rwsem); +} + +static void cdma_tbl_init(struct cdma_table *table, u32 max, u32 min) +{ + if (!max || max < min) + return; + + spin_lock_init(&table->lock); + idr_init(&table->idr_tbl.idr); + table->idr_tbl.max = max; + table->idr_tbl.min = min; + table->idr_tbl.next = min; +} + +static void cdma_tbl_destroy(struct cdma_dev *cdev, struct cdma_table *table, + const char *table_name) +{ + if (!idr_is_empty(&table->idr_tbl.idr)) + dev_err(cdev->dev, "IDR not empty in clean up %s table.\n", + table_name); + idr_destroy(&table->idr_tbl.idr); +} + +static void cdma_init_tables(struct cdma_dev *cdev) +{ + struct cdma_res *queue = &cdev->caps.queue; + struct cdma_res *jfce = &cdev->caps.jfce; + struct cdma_res *jfs = &cdev->caps.jfs; + struct cdma_res *jfc = &cdev->caps.jfc; + + cdma_tbl_init(&cdev->queue_table, queue->start_idx + queue->max_cnt - 1, + queue->start_idx); + cdma_tbl_init(&cdev->jfce_table, jfce->start_idx + jfce->max_cnt - 1, + jfce->start_idx); + cdma_tbl_init(&cdev->jfc_table, jfc->start_idx + jfc->max_cnt - 1, + jfc->start_idx); + cdma_tbl_init(&cdev->jfs_table, jfs->max_cnt + jfs->start_idx - 1, + jfs->start_idx); + cdma_tbl_init(&cdev->ctp_table, CDMA_RANGE_INDEX_ENTRY_CNT, 0); + cdma_tbl_init(&cdev->seg_table, CDMA_SEGMENT_ENTRY_CNT, 0); +} + +static void cdma_destroy_tables(struct cdma_dev *cdev) +{ + cdma_tbl_destroy(cdev, &cdev->seg_table, "SEG"); + cdma_tbl_destroy(cdev, &cdev->ctp_table, "CTP"); + cdma_tbl_destroy(cdev, &cdev->jfs_table, "JFS"); + cdma_tbl_destroy(cdev, &cdev->jfc_table, "JFC"); + cdma_tbl_destroy(cdev, &cdev->jfce_table, "JFCE"); + cdma_tbl_destroy(cdev, &cdev->queue_table, "QUEUE"); +} + +static void cdma_init_base_dev(struct cdma_dev *cdev) +{ + struct cdma_device_attr *attr = &cdev->base.attr; + struct cdma_device_cap *dev_cap = &attr->dev_cap; + struct cdma_caps *caps = &cdev->caps; + + attr->eid.dw0 = cdev->eid; + dev_cap->max_jfc = caps->jfc.max_cnt; + dev_cap->max_jfs = caps->jfs.max_cnt; + dev_cap->max_jfc_depth = caps->jfc.depth; + dev_cap->max_jfs_depth = caps->jfs.depth; + dev_cap->trans_mode = caps->trans_mode; + dev_cap->max_jfs_sge = caps->jfs_sge; + dev_cap->max_jfs_rsge = caps->jfs_rsge; + dev_cap->max_msg_size = caps->max_msg_len; + dev_cap->ceq_cnt = caps->comp_vector_cnt; + dev_cap->max_jfs_inline_len = caps->jfs_inline_sz; +} + +static int cdma_init_dev_param(struct cdma_dev *cdev, + struct auxiliary_device *adev) +{ + struct ubase_resource_space *mem_base; + int ret; + + mem_base = ubase_get_mem_base(adev); + if (!mem_base) + return -EINVAL; + + cdev->adev = adev; + cdev->dev = adev->dev.parent; + cdev->k_db_base = mem_base->addr; + cdev->db_base = mem_base->addr_unmapped; + + ret = cdma_init_dev_caps(cdev); + if (ret) + return ret; + + cdma_init_base_dev(cdev); + cdma_init_tables(cdev); + + dev_set_drvdata(&adev->dev, cdev); + + mutex_init(&cdev->db_mutex); + mutex_init(&cdev->eu_mutex); + INIT_LIST_HEAD(&cdev->db_page); + mutex_init(&cdev->file_mutex); + INIT_LIST_HEAD(&cdev->file_list); + + return 0; +} + +static void cdma_uninit_dev_param(struct cdma_dev *cdev) +{ + mutex_destroy(&cdev->db_mutex); + mutex_destroy(&cdev->eu_mutex); + mutex_destroy(&cdev->file_mutex); + dev_set_drvdata(&cdev->adev->dev, NULL); + cdma_destroy_tables(cdev); +} + +static void cdma_release_table_res(struct cdma_dev *cdev) +{ + struct cdma_queue *queue; + struct cdma_segment *seg; + struct cdma_jfc *jfc; + struct cdma_jfs *jfs; + struct cdma_tp *tmp; + int id; + + idr_for_each_entry(&cdev->ctp_table.idr_tbl.idr, tmp, id) + cdma_destroy_ctp_imm(cdev, tmp->base.tp_id); + + idr_for_each_entry(&cdev->jfs_table.idr_tbl.idr, jfs, id) + cdma_delete_jfs(cdev, jfs->id); + + idr_for_each_entry(&cdev->jfc_table.idr_tbl.idr, jfc, id) + cdma_delete_jfc(cdev, jfc->jfcn, NULL); + + idr_for_each_entry(&cdev->queue_table.idr_tbl.idr, queue, id) + cdma_delete_queue(cdev, queue->id); + + idr_for_each_entry(&cdev->seg_table.idr_tbl.idr, seg, id) + cdma_unregister_seg(cdev, seg); +} + +static int cdma_ctrlq_eu_add(struct cdma_dev *cdev, struct eu_info *eu) +{ + struct cdma_device_attr *attr = &cdev->base.attr; + struct eu_info *eus = cdev->base.attr.eus; + u8 i; + + for (i = 0; i < attr->eu_num; i++) { + if (eu->eid_idx != eus[i].eid_idx) + continue; + + dev_dbg(cdev->dev, + "cdma.%u: eid_idx[0x%x] eid[0x%x->0x%x] upi[0x%x->0x%x] update success.\n", + cdev->adev->id, eu->eid_idx, eus[i].eid.dw0, + eu->eid.dw0, eus[i].upi, eu->upi & CDMA_UPI_MASK); + + eus[i].eid = eu->eid; + eus[i].upi = eu->upi & CDMA_UPI_MASK; + + if (attr->eu.eid_idx == eu->eid_idx) { + attr->eu.eid = eu->eid; + attr->eu.upi = eu->upi & CDMA_UPI_MASK; + } + return 0; + } + + if (attr->eu_num >= CDMA_MAX_EU_NUM) { + dev_err(cdev->dev, "cdma.%u: eu table is full.\n", + cdev->adev->id); + return -EINVAL; + } + + eus[attr->eu_num++] = *eu; + dev_dbg(cdev->dev, + "cdma.%u: eid_idx[0x%x] eid[0x%x] upi[0x%x] add success.\n", + cdev->adev->id, eu->eid_idx, eu->eid.dw0, + eu->upi & CDMA_UPI_MASK); + + return 0; +} + +static int cdma_ctrlq_eu_del(struct cdma_dev *cdev, struct eu_info *eu) +{ + struct cdma_device_attr *attr = &cdev->base.attr; + struct eu_info *eus = cdev->base.attr.eus; + int ret = -EINVAL; + u8 i, j; + + if (!attr->eu_num) { + dev_err(cdev->dev, "cdma.%u: eu table is empty.\n", + cdev->adev->id); + return -EINVAL; + } + + for (i = 0; i < attr->eu_num; i++) { + if (eu->eid_idx != eus[i].eid_idx) + continue; + + for (j = i; j < attr->eu_num - 1; j++) + eus[j] = eus[j + 1]; + memset(&eus[j], 0, sizeof(*eus)); + + if (attr->eu.eid_idx == eu->eid_idx) + attr->eu = eus[0]; + attr->eu_num--; + ret = 0; + break; + } + + dev_info(cdev->dev, + "cdma.%u: eid_idx[0x%x] eid[0x%x] upi[0x%x] delete %s.\n", + cdev->adev->id, eu->eid_idx, eu->eid.dw0, + eu->upi & CDMA_UPI_MASK, ret ? "failed" : "success"); + + return ret; +} + +static int cdma_ctrlq_eu_update(struct auxiliary_device *adev, u8 service_ver, + void *data, u16 len, u16 seq) +{ + struct cdma_dev *cdev = dev_get_drvdata(&adev->dev); + struct cdma_ctrlq_eu_info *ctrlq_eu; + int ret = -EINVAL; + + if (len < sizeof(*ctrlq_eu)) { + dev_err(cdev->dev, "ctrlq data len is invalid.\n"); + return -EINVAL; + } + + ctrlq_eu = (struct cdma_ctrlq_eu_info *)data; + + mutex_lock(&cdev->eu_mutex); + if (ctrlq_eu->op == CDMA_CTRLQ_EU_ADD) + ret = cdma_ctrlq_eu_add(cdev, &ctrlq_eu->eu); + else if (ctrlq_eu->op == CDMA_CTRLQ_EU_DEL) + ret = cdma_ctrlq_eu_del(cdev, &ctrlq_eu->eu); + else + dev_err(cdev->dev, "ctrlq eu op is invalid.\n"); + mutex_unlock(&cdev->eu_mutex); + + return ret; +} + +int cdma_create_arm_db_page(struct cdma_dev *cdev) +{ + cdev->arm_db_page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!cdev->arm_db_page) { + dev_err(cdev->dev, "alloc dev arm db page failed.\n"); + return -ENOMEM; + } + return 0; +} + +void cdma_destroy_arm_db_page(struct cdma_dev *cdev) +{ + if (!cdev->arm_db_page) + return; + + put_page(cdev->arm_db_page); + cdev->arm_db_page = NULL; +} + +int cdma_register_crq_event(struct auxiliary_device *adev) +{ + struct ubase_ctrlq_event_nb nb = { + .service_type = UBASE_CTRLQ_SER_TYPE_DEV_REGISTER, + .opcode = CDMA_CTRLQ_EU_UPDATE, + .back = adev, + .crq_handler = cdma_ctrlq_eu_update, + }; + int ret; + + if (!adev) + return -EINVAL; + + ret = ubase_ctrlq_register_crq_event(adev, &nb); + if (ret) { + dev_err(&adev->dev, "register crq event failed, id = %u, ret = %d.\n", + adev->id, ret); + return ret; + } + + return 0; +} + +void cdma_unregister_crq_event(struct auxiliary_device *adev) +{ + ubase_ctrlq_unregister_crq_event(adev, + UBASE_CTRLQ_SER_TYPE_DEV_REGISTER, + CDMA_CTRLQ_EU_UPDATE); +} + +struct cdma_dev *cdma_create_dev(struct auxiliary_device *adev) +{ + struct cdma_dev *cdev; + + cdev = kzalloc((sizeof(*cdev)), GFP_KERNEL); + if (!cdev) + return NULL; + + if (cdma_init_dev_param(cdev, adev)) + goto free; + + if (cdma_add_device_to_list(cdev)) + goto free_param; + + if (cdma_alloc_dev_tid(cdev)) + goto del_list; + + if (cdma_register_crq_event(adev)) + goto free_tid; + + if (cdma_create_arm_db_page(cdev)) + goto unregister_crq; + + idr_init(&cdev->ctx_idr); + spin_lock_init(&cdev->ctx_lock); + atomic_set(&cdev->cmdcnt, 1); + init_completion(&cdev->cmddone); + + dev_dbg(&adev->dev, "cdma.%u init succeeded.\n", adev->id); + + return cdev; + +unregister_crq: + cdma_unregister_crq_event(adev); +free_tid: + cdma_free_dev_tid(cdev); +del_list: + cdma_del_device_from_list(cdev); +free_param: + cdma_uninit_dev_param(cdev); +free: + kfree(cdev); + return NULL; +} + +void cdma_destroy_dev(struct cdma_dev *cdev, bool is_remove) +{ + struct cdma_context *tmp; + int id; + + if (!cdev) + return; + + ubase_virt_unregister(cdev->adev); + + if (is_remove) { + cdma_release_table_res(cdev); + + idr_for_each_entry(&cdev->ctx_idr, tmp, id) + cdma_free_context(cdev, tmp); + idr_destroy(&cdev->ctx_idr); + } + + cdma_destroy_arm_db_page(cdev); + ubase_ctrlq_unregister_crq_event(cdev->adev, + UBASE_CTRLQ_SER_TYPE_DEV_REGISTER, + CDMA_CTRLQ_EU_UPDATE); + + if (is_remove) { + cdma_free_dev_tid(cdev); + + cdma_del_device_from_list(cdev); + cdma_uninit_dev_param(cdev); + kfree(cdev); + } +} + +bool cdma_find_seid_in_eus(struct eu_info *eus, u8 eu_num, struct dev_eid *eid, + struct eu_info *eu_out) +{ + u32 i; + + for (i = 0; i < eu_num; i++) + if (eus[i].eid.dw0 == eid->dw0 && eus[i].eid.dw1 == eid->dw1 && + eus[i].eid.dw2 == eid->dw2 && eus[i].eid.dw3 == eid->dw3) { + *eu_out = eus[i]; + return true; + } + + return false; +} diff --git a/drivers/ub/cdma/cdma_dev.h b/drivers/ub/cdma/cdma_dev.h new file mode 100644 index 0000000000000000000000000000000000000000..d433218934f1de58f6ac9ac69a82061a0c706590 --- /dev/null +++ b/drivers/ub/cdma/cdma_dev.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_DEV_H__ +#define __CDMA_DEV_H__ + +#include +#include + +#define CDMA_CTRLQ_EU_UPDATE 0x2 +#define CDMA_UE_MAX_NUM 64 + +struct cdma_dev; + +struct cdma_ctrlq_eu_info { + struct eu_info eu; + u32 op : 4; + u32 rsvd : 28; +}; + +enum cdma_ctrlq_eu_op { + CDMA_CTRLQ_EU_ADD = 0, + CDMA_CTRLQ_EU_DEL = 1, +}; + +struct cdma_dev *cdma_create_dev(struct auxiliary_device *adev); +void cdma_destroy_dev(struct cdma_dev *cdev, bool is_remove); +struct cdma_dev *get_cdma_dev_by_eid(u32 eid); +struct xarray *get_cdma_dev_tbl(u32 *devices_num); +bool cdma_find_seid_in_eus(struct eu_info *eus, u8 eu_num, struct dev_eid *eid, + struct eu_info *eu_out); +int cdma_register_crq_event(struct auxiliary_device *adev); +void cdma_unregister_crq_event(struct auxiliary_device *adev); +int cdma_create_arm_db_page(struct cdma_dev *cdev); +void cdma_destroy_arm_db_page(struct cdma_dev *cdev); + +#endif /* _CDMA_DEV_H_ */ diff --git a/drivers/ub/cdma/cdma_eq.c b/drivers/ub/cdma/cdma_eq.c new file mode 100644 index 0000000000000000000000000000000000000000..6bc6048e31271637238c9ad1b653f85a49ab9995 --- /dev/null +++ b/drivers/ub/cdma/cdma_eq.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "CDMA: " fmt + +#include +#include +#include +#include +#include "cdma_jfs.h" +#include "cdma_jfc.h" +#include "cdma.h" +#include "cdma_eq.h" + +static int cdma_ae_jfs_check_error(struct auxiliary_device *adev, + u32 jetty_id) +{ + struct cdma_dev *cdev = get_cdma_dev(adev); + struct cdma_base_jfs *base_jfs; + struct cdma_event ae; + struct cdma_jfs *jfs; + + spin_lock(&cdev->jfs_table.lock); + jfs = idr_find(&cdev->jfs_table.idr_tbl.idr, jetty_id); + if (!jfs) { + dev_err(cdev->dev, "ae get jfs from table failed, id = %u.\n", + jetty_id); + spin_unlock(&cdev->jfs_table.lock); + return -EINVAL; + } + + base_jfs = &jfs->base_jfs; + + if (base_jfs->jfae_handler && base_jfs->ctx) { + refcount_inc(&jfs->ae_ref_cnt); + spin_unlock(&cdev->jfs_table.lock); + ae.dev = base_jfs->dev; + ae.element.jfs = base_jfs; + ae.event_type = CDMA_EVENT_JFS_ERR; + base_jfs->jfae_handler(&ae, base_jfs->ctx); + if (refcount_dec_and_test(&jfs->ae_ref_cnt)) { + complete(&jfs->ae_comp); + dev_dbg(cdev->dev, "jfs ae handler done.\n"); + } + } else { + spin_unlock(&cdev->jfs_table.lock); + } + + return 0; +} + +static int cdma_ae_jfc_check_error(struct auxiliary_device *adev, + u32 jetty_id) +{ + struct cdma_dev *cdev = get_cdma_dev(adev); + struct cdma_base_jfc *base_jfc; + struct cdma_event ae; + struct cdma_jfc *jfc; + unsigned long flags; + + spin_lock_irqsave(&cdev->jfc_table.lock, flags); + jfc = idr_find(&cdev->jfc_table.idr_tbl.idr, jetty_id); + if (!jfc) { + dev_err(cdev->dev, "get jfc from table failed, id = %u.\n", + jetty_id); + spin_unlock_irqrestore(&cdev->jfc_table.lock, flags); + return -EINVAL; + } + base_jfc = &jfc->base; + + if (base_jfc->jfae_handler && base_jfc->ctx) { + refcount_inc(&jfc->event_refcount); + spin_unlock_irqrestore(&cdev->jfc_table.lock, flags); + ae.dev = base_jfc->dev; + ae.element.jfc = base_jfc; + ae.event_type = CDMA_EVENT_JFC_ERR; + base_jfc->jfae_handler(&ae, base_jfc->ctx); + if (refcount_dec_and_test(&jfc->event_refcount)) { + complete(&jfc->event_comp); + dev_dbg(cdev->dev, "jfc ae handler done.\n"); + } + } else { + spin_unlock_irqrestore(&cdev->jfc_table.lock, flags); + } + + return 0; +} + +static int cdma_ae_jetty_level_error(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct ubase_event_nb *ev_nb = container_of(nb, struct ubase_event_nb, nb); + struct auxiliary_device *adev = ev_nb->back; + struct ubase_aeq_notify_info *info = data; + u32 jetty_id; + + jetty_id = info->aeqe->event.queue_event.num; + + switch (info->sub_type) { + case UBASE_SUBEVENT_TYPE_JFS_CHECK_ERROR: + return cdma_ae_jfs_check_error(adev, jetty_id); + case UBASE_SUBEVENT_TYPE_JFC_CHECK_ERROR: + return cdma_ae_jfc_check_error(adev, jetty_id); + default: + dev_warn(&adev->dev, "cdma get unsupported async event type %u.\n", + info->sub_type); + return -EINVAL; + } +} + +static struct cdma_ae_operation cdma_ae_opts[] = { + {UBASE_EVENT_TYPE_JETTY_LEVEL_ERROR, cdma_ae_jetty_level_error} +}; + +static int cdma_event_register(struct auxiliary_device *adev, + enum ubase_event_type event_type, notifier_fn_t call) +{ + struct cdma_dev *cdma_dev = get_cdma_dev(adev); + struct ubase_event_nb *event_cb; + int ret; + + event_cb = kzalloc(sizeof(*event_cb), GFP_KERNEL); + if (!event_cb) + return -ENOMEM; + + event_cb->drv_type = UBASE_DRV_CDMA; + event_cb->event_type = event_type; + event_cb->back = (void *)adev; + event_cb->nb.notifier_call = call; + + ret = ubase_event_register(adev, event_cb); + if (ret) { + dev_err(cdma_dev->dev, + "register async event failed, event type = %u, ret = %d.\n", + event_cb->event_type, ret); + kfree(event_cb); + return ret; + } + cdma_dev->ae_event_addr[event_type] = event_cb; + + return 0; +} + +/* thanks to drivers/infiniband/hw/erdma/erdma_eq.c */ +int cdma_reg_ae_event(struct auxiliary_device *adev) +{ + struct cdma_dev *cdma_dev; + u32 opt_num; + int ret = 0; + int i; + + if (!adev) + return -EINVAL; + + cdma_dev = get_cdma_dev(adev); + if (!cdma_dev) + return -EINVAL; + + opt_num = sizeof(cdma_ae_opts) / sizeof(struct cdma_ae_operation); + for (i = 0; i < opt_num; ++i) { + ret = cdma_event_register(adev, + (enum ubase_event_type)cdma_ae_opts[i].op_code, + cdma_ae_opts[i].call); + if (ret) { + cdma_unreg_ae_event(adev); + return -EINVAL; + } + } + + dev_dbg(cdma_dev->dev, "cdma register ae event, ret = %d.\n", ret); + + return ret; +} + +void cdma_unreg_ae_event(struct auxiliary_device *adev) +{ + struct cdma_dev *cdma_dev; + int i; + + if (!adev) + return; + + cdma_dev = get_cdma_dev(adev); + if (!cdma_dev) + return; + + for (i = 0; i < UBASE_EVENT_TYPE_MAX; i++) { + if (cdma_dev->ae_event_addr[i]) { + ubase_event_unregister(adev, cdma_dev->ae_event_addr[i]); + kfree(cdma_dev->ae_event_addr[i]); + cdma_dev->ae_event_addr[i] = NULL; + } + } +} + +/* thanks to drivers/infiniband/hw/erdma/erdma_eq.c */ +int cdma_reg_ce_event(struct auxiliary_device *adev) +{ + struct cdma_dev *cdma_dev; + int ret; + + if (!adev) + return -EINVAL; + + cdma_dev = get_cdma_dev(adev); + if (!cdma_dev) + return -EINVAL; + + ret = ubase_comp_register(adev, cdma_jfc_completion); + if (ret) + dev_err(cdma_dev->dev, + "register ce event failed, ret = %d.\n", ret); + + return ret; +} + +void cdma_unreg_ce_event(struct auxiliary_device *adev) +{ + if (!adev) + return; + + ubase_comp_unregister(adev); +} diff --git a/drivers/ub/cdma/cdma_eq.h b/drivers/ub/cdma/cdma_eq.h new file mode 100644 index 0000000000000000000000000000000000000000..70e9edcccad402f4b83c718f5eb694bc8919564b --- /dev/null +++ b/drivers/ub/cdma/cdma_eq.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_EQ_H__ +#define __CDMA_EQ_H__ +#include + +struct cdma_ae_operation { + u32 op_code; + notifier_fn_t call; +}; + +int cdma_reg_ae_event(struct auxiliary_device *adev); +void cdma_unreg_ae_event(struct auxiliary_device *adev); +int cdma_reg_ce_event(struct auxiliary_device *adev); +void cdma_unreg_ce_event(struct auxiliary_device *adev); + +#endif diff --git a/drivers/ub/cdma/cdma_event.c b/drivers/ub/cdma/cdma_event.c new file mode 100644 index 0000000000000000000000000000000000000000..057bf2daefc301bce9cac8092dc9f0c8073aed97 --- /dev/null +++ b/drivers/ub/cdma/cdma_event.c @@ -0,0 +1,782 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define pr_fmt(fmt) "CDMA: " fmt +#define dev_fmt pr_fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include "cdma_uobj.h" +#include "cdma_event.h" + +static __poll_t cdma_jfe_poll(struct cdma_jfe *jfe, struct file *filp, + struct poll_table_struct *wait) +{ + __poll_t flag = 0; + + poll_wait(filp, &jfe->poll_wait, wait); + + spin_lock_irq(&jfe->lock); + if (!list_empty(&jfe->event_list)) + flag = EPOLLIN | EPOLLRDNORM; + + spin_unlock_irq(&jfe->lock); + + return flag; +} + +static u32 cdma_read_jfe_event(struct cdma_jfe *jfe, u32 max_event_cnt, + struct list_head *event_list) +{ + struct cdma_jfe_event *event; + struct list_head *next; + struct list_head *p; + u32 cnt = 0; + + if (!max_event_cnt) + return 0; + + spin_lock_irq(&jfe->lock); + + list_for_each_safe(p, next, &jfe->event_list) { + event = list_entry(p, struct cdma_jfe_event, node); + if (event->counter) { + ++(*event->counter); + list_del(&event->obj_node); + } + list_del(p); + if (jfe->event_list_count > 0) + jfe->event_list_count--; + list_add_tail(p, event_list); + cnt++; + if (cnt == max_event_cnt) + break; + } + spin_unlock_irq(&jfe->lock); + + return cnt; +} + +static int cdma_wait_event(struct cdma_jfe *jfe, bool nonblock, + u32 max_event_cnt, u32 *event_cnt, + struct list_head *event_list) +{ + int ret; + + *event_cnt = 0; + spin_lock_irq(&jfe->lock); + while (list_empty(&jfe->event_list)) { + spin_unlock_irq(&jfe->lock); + if (nonblock) + return -EAGAIN; + + ret = wait_event_interruptible(jfe->poll_wait, + !list_empty(&jfe->event_list)); + if (ret) + return ret; + + spin_lock_irq(&jfe->lock); + if (list_empty(&jfe->event_list)) { + spin_unlock_irq(&jfe->lock); + return -EIO; + } + } + spin_unlock_irq(&jfe->lock); + *event_cnt = cdma_read_jfe_event(jfe, max_event_cnt, event_list); + + return 0; +} + +static int cdma_wait_event_timeout(struct cdma_jfe *jfe, + unsigned long max_timeout, + u32 max_event_cnt, + u32 *event_cnt, + struct list_head *event_list) +{ + long timeout = (long)max_timeout; + + *event_cnt = 0; + while (1) { + asm volatile("" : : : "memory"); + *event_cnt = cdma_read_jfe_event(jfe, max_event_cnt, event_list); + if (*event_cnt > 0) + break; + timeout = wait_event_interruptible_timeout(jfe->poll_wait, + !list_empty(&jfe->event_list), timeout); + if (timeout <= 0) + return timeout; + } + + return 0; +} + +static int cdma_jfce_wait(struct cdma_jfce *jfce, struct file *filp, + unsigned long arg) +{ + struct cdma_cmd_jfce_wait_args we = { 0 }; + struct cdma_jfe_event *event; + struct list_head event_list; + struct list_head *next; + struct list_head *p; + u32 max_event_cnt; + u32 i = 0; + int ret; + + if (copy_from_user(&we, (const void __user *)arg, + (u32)sizeof(we)) != 0) + return -EFAULT; + + max_event_cnt = min_t(u32, we.in.max_event_cnt, (u32)CDMA_MAX_JFCE_EVENT_CNT); + INIT_LIST_HEAD(&event_list); + if (we.in.time_out <= 0) { + ret = cdma_wait_event(&jfce->jfe, + (filp->f_flags & O_NONBLOCK) | + (!we.in.time_out), + max_event_cnt, + &we.out.event_cnt, &event_list); + } else { + ret = cdma_wait_event_timeout(&jfce->jfe, + msecs_to_jiffies(we.in.time_out), + max_event_cnt, &we.out.event_cnt, + &event_list); + } + + if (ret < 0) { + pr_err("wait jfce event failed, ret = %d\n", ret); + return ret; + } + + list_for_each_safe(p, next, &event_list) { + event = list_entry(p, struct cdma_jfe_event, node); + we.out.event_data[i++] = event->event_data; + list_del(p); + kfree(event); + } + + if (we.out.event_cnt > 0 && copy_to_user((void *)arg, &we, sizeof(we))) { + pr_err("copy to user failed.\n"); + return -EFAULT; + } + + return 0; +} + +static __poll_t cdma_jfce_poll(struct file *filp, struct poll_table_struct *wait) +{ + struct cdma_jfce *jfce = (struct cdma_jfce *)filp->private_data; + + if (!jfce) + return POLLERR; + + return cdma_jfe_poll(&jfce->jfe, filp, wait); +} + +static long cdma_jfce_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) +{ + struct cdma_jfce *jfce = (struct cdma_jfce *)filp->private_data; + unsigned int nr; + int ret; + + if (!arg || !jfce || _IOC_TYPE(cmd) != CDMA_EVENT_CMD_MAGIC) { + pr_err("invalid parameter, cmd = %u.\n", cmd); + return -EINVAL; + } + + nr = (unsigned int)_IOC_NR(cmd); + switch (nr) { + case JFCE_CMD_WAIT_EVENT: + ret = cdma_jfce_wait(jfce, filp, arg); + break; + default: + ret = -ENOIOCTLCMD; + break; + } + + return ret; +} + +static int cdma_delete_jfce(struct inode *inode, struct file *filp) +{ + struct cdma_file *cfile; + struct cdma_jfce *jfce; + + if (!filp || !filp->private_data) + return 0; + + jfce = (struct cdma_jfce *)filp->private_data; + + cfile = jfce->cfile; + if (!cfile) + return 0; + + if (!mutex_trylock(&cfile->ctx_mutex)) + return -ENOLCK; + cdma_destroy_jfce(jfce); + filp->private_data = NULL; + mutex_unlock(&cfile->ctx_mutex); + cdma_close_uobj_fd(cfile); + + pr_info("jfce is release.\n"); + return 0; +} + +static int cdma_jfce_fasync(int fd, struct file *filp, int on) +{ + struct cdma_jfce *jfce = (struct cdma_jfce *)filp->private_data; + int ret; + + if (!jfce) + return -EINVAL; + + spin_lock_irq(&jfce->jfe.lock); + ret = fasync_helper(fd, filp, on, &jfce->jfe.async_queue); + spin_unlock_irq(&jfce->jfe.lock); + + return ret; +} + +const struct file_operations cdma_jfce_fops = { + .owner = THIS_MODULE, + .poll = cdma_jfce_poll, + .unlocked_ioctl = cdma_jfce_ioctl, + .release = cdma_delete_jfce, + .fasync = cdma_jfce_fasync, +}; + +static int cdma_jfce_id_alloc(struct cdma_dev *cdev, struct cdma_jfce *jfce) +{ + struct cdma_table *jfce_tbl = &cdev->jfce_table; + int id; + + idr_preload(GFP_KERNEL); + spin_lock(&jfce_tbl->lock); + id = idr_alloc(&jfce_tbl->idr_tbl.idr, jfce, jfce_tbl->idr_tbl.min, + jfce_tbl->idr_tbl.max, GFP_NOWAIT); + if (id < 0) + dev_err(cdev->dev, "alloc jfce id failed.\n"); + spin_unlock(&jfce_tbl->lock); + idr_preload_end(); + + return id; +} + +static void cdma_jfce_id_free(struct cdma_dev *cdev, u32 jfce_id) +{ + struct cdma_table *jfce_tbl = &cdev->jfce_table; + + spin_lock(&jfce_tbl->lock); + idr_remove(&jfce_tbl->idr_tbl.idr, jfce_id); + spin_unlock(&jfce_tbl->lock); +} + +static void cdma_write_event(struct cdma_jfe *jfe, u64 event_data, + u32 event_type, struct list_head *obj_event_list, + u32 *counter) +{ + struct cdma_jfe_event *event; + unsigned long flags; + + event = kzalloc(sizeof(*event), GFP_ATOMIC); + if (event == NULL) + return; + + spin_lock_irqsave(&jfe->lock, flags); + INIT_LIST_HEAD(&event->obj_node); + event->event_type = event_type; + event->event_data = event_data; + event->counter = counter; + list_add_tail(&event->node, &jfe->event_list); + if (obj_event_list) + list_add_tail(&event->obj_node, obj_event_list); + if (jfe->async_queue) + kill_fasync(&jfe->async_queue, SIGIO, POLL_IN); + jfe->event_list_count++; + spin_unlock_irqrestore(&jfe->lock, flags); + wake_up_interruptible(&jfe->poll_wait); +} + +static void cdma_init_jfe(struct cdma_jfe *jfe) +{ + spin_lock_init(&jfe->lock); + INIT_LIST_HEAD(&jfe->event_list); + init_waitqueue_head(&jfe->poll_wait); + jfe->async_queue = NULL; + jfe->event_list_count = 0; +} + +static void cdma_uninit_jfe(struct cdma_jfe *jfe) +{ + struct cdma_jfe_event *event; + struct list_head *p, *next; + + spin_lock_irq(&jfe->lock); + list_for_each_safe(p, next, &jfe->event_list) { + event = list_entry(p, struct cdma_jfe_event, node); + if (event->counter) + list_del(&event->obj_node); + kfree(event); + } + spin_unlock_irq(&jfe->lock); +} + +struct cdma_jfce *cdma_get_jfce_from_id(struct cdma_dev *cdev, int jfce_id) +{ + struct cdma_table *jfce_table = &cdev->jfce_table; + struct cdma_jfce *jfce; + struct file *file; + + spin_lock(&jfce_table->lock); + jfce = idr_find(&jfce_table->idr_tbl.idr, jfce_id); + if (!jfce) { + dev_err(cdev->dev, "find jfce failed, id = %d.\n", jfce_id); + } else { + file = fget(jfce->fd); + if (!file) { + jfce = NULL; + } else { + if (file->private_data != jfce) { + fput(file); + jfce = NULL; + } + } + } + spin_unlock(&jfce_table->lock); + + return jfce; +} + +void cdma_jfc_comp_event_cb(struct cdma_base_jfc *jfc) +{ + struct cdma_jfc_event *jfc_event; + struct cdma_jfce *jfce; + + if (!jfc) + return; + + jfc_event = &jfc->jfc_event; + if (!IS_ERR_OR_NULL(jfc_event->jfce)) { + jfce = jfc_event->jfce; + if (jfce->jfe.event_list_count >= MAX_EVENT_LIST_SIZE) + return; + + cdma_write_event(&jfce->jfe, jfc->jfc_cfg.queue_id, 0, + &jfc_event->comp_event_list, + &jfc_event->comp_events_reported); + } +} + +struct cdma_jfce *cdma_alloc_jfce(struct cdma_file *cfile) +{ + struct cdma_jfce *jfce; + struct file *file; + int new_fd; + int ret; + + if (!cfile) + return ERR_PTR(-EINVAL); + + new_fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); + if (new_fd < 0) + return ERR_PTR(new_fd); + + jfce = kzalloc(sizeof(*jfce), GFP_KERNEL); + if (!jfce) { + ret = -ENOMEM; + goto err_put_unused_fd; + } + + ret = cdma_jfce_id_alloc(cfile->cdev, jfce); + if (ret < 0) + goto err_free_jfce; + jfce->id = ret; + + file = anon_inode_getfile("[jfce]", &cdma_jfce_fops, jfce, + O_RDWR | O_CLOEXEC); + if (IS_ERR(file)) { + ret = PTR_ERR(file); + goto err_free_id; + } + + cdma_init_jfe(&jfce->jfe); + jfce->cdev = cfile->cdev; + jfce->fd = new_fd; + jfce->file = file; + jfce->cfile = cfile; + kref_get(&cfile->ref); + fd_install(new_fd, file); + + return jfce; + +err_free_id: + cdma_jfce_id_free(cfile->cdev, jfce->id); +err_free_jfce: + kfree(jfce); +err_put_unused_fd: + put_unused_fd(new_fd); + + return ERR_PTR(ret); +} + +void cdma_free_jfce(struct cdma_jfce *jfce) +{ + struct cdma_dev *cdev; + + if (!jfce || !jfce->cdev) + return; + + cdev = jfce->cdev; + + if (jfce->id >= cdev->caps.jfce.max_cnt + cdev->caps.jfce.start_idx || + jfce->id < cdev->caps.jfce.start_idx) { + dev_err(cdev->dev, + "jfce id invalid, id = %u, start_idx = %u, max_cnt = %u.\n", + jfce->id, cdev->caps.jfce.start_idx, + cdev->caps.jfce.max_cnt); + return; + } + + fput(jfce->file); + put_unused_fd(jfce->fd); +} + +void cdma_destroy_jfce(struct cdma_jfce *jfce) +{ + if (!jfce) + return; + + cdma_uninit_jfe(&jfce->jfe); + if (jfce->cfile && jfce->cfile->cdev) + cdma_jfce_id_free(jfce->cdev, jfce->id); + kfree(jfce); +} + +static void cdma_write_async_event(struct cdma_context *ctx, u64 event_data, + u32 type, struct list_head *obj_event_list, + u32 *counter) +{ + struct cdma_jfae *jfae; + + rcu_read_lock(); + jfae = (struct cdma_jfae *)(rcu_dereference(ctx->jfae)); + if (!jfae) + goto err_free_rcu; + + if (jfae->jfe.event_list_count >= MAX_EVENT_LIST_SIZE) { + pr_debug("event list overflow, and this write will be discarded.\n"); + goto err_free_rcu; + } + + cdma_write_event(&jfae->jfe, event_data, type, obj_event_list, counter); + +err_free_rcu: + rcu_read_unlock(); +} + +void cdma_jfs_async_event_cb(struct cdma_event *event, struct cdma_context *ctx) +{ + struct cdma_jfs_event *jfs_event; + + jfs_event = &event->element.jfs->jfs_event; + cdma_write_async_event(ctx, event->element.jfs->cfg.queue_id, + event->event_type, &jfs_event->async_event_list, + &jfs_event->async_events_reported); +} + +void cdma_jfc_async_event_cb(struct cdma_event *event, struct cdma_context *ctx) +{ + struct cdma_jfc_event *jfc_event; + + jfc_event = &event->element.jfc->jfc_event; + cdma_write_async_event(ctx, event->element.jfc->jfc_cfg.queue_id, + event->event_type, &jfc_event->async_event_list, + &jfc_event->async_events_reported); +} + +static inline void cdma_set_async_event(struct cdma_cmd_async_event *async_event, + const struct cdma_jfe_event *event) +{ + async_event->event_data = event->event_data; + async_event->event_type = event->event_type; +} + +static int cdma_get_async_event(struct cdma_jfae *jfae, struct file *filp, + unsigned long arg) +{ + struct cdma_cmd_async_event async_event = { 0 }; + struct cdma_jfe_event *event; + struct list_head event_list; + u32 event_cnt; + int ret; + + if (!arg) { + pr_err("invalid jfae arg.\n"); + return -EINVAL; + } + + if (!jfae->cfile->cdev || jfae->cfile->cdev->status == CDMA_SUSPEND) { + pr_info("wait dev invalid event success.\n"); + async_event.event_data = 0; + async_event.event_type = CDMA_EVENT_DEV_INVALID; + ret = (int)copy_to_user((void *)arg, &async_event, + sizeof(async_event)); + if (ret) { + pr_err("dev copy to user failed, ret = %d\n", ret); + return -EFAULT; + } + } else { + INIT_LIST_HEAD(&event_list); + ret = cdma_wait_event(&jfae->jfe, filp->f_flags & O_NONBLOCK, 1, + &event_cnt, &event_list); + if (ret < 0) { + pr_err("wait event failed, ret = %d.\n", ret); + return ret; + } + event = list_first_entry(&event_list, struct cdma_jfe_event, node); + if (event == NULL) + return -EIO; + + cdma_set_async_event(&async_event, event); + list_del(&event->node); + kfree(event); + + if (event_cnt > 0) { + ret = (int)copy_to_user((void *)arg, &async_event, + sizeof(async_event)); + if (ret) { + pr_err("dev copy to user failed, ret = %d\n", ret); + return -EFAULT; + } + } + } + + return 0; +} + +static __poll_t cdma_jfae_poll(struct file *filp, struct poll_table_struct *wait) +{ + struct cdma_jfae *jfae = (struct cdma_jfae *)filp->private_data; + + if (!jfae || !jfae->cfile || !jfae->cfile->cdev) + return POLLERR; + + if (jfae->cfile->cdev->status == CDMA_SUSPEND) + return POLLIN | POLLRDNORM; + + return cdma_jfe_poll(&jfae->jfe, filp, wait); +} + +static long cdma_jfae_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct cdma_jfae *jfae = (struct cdma_jfae *)filp->private_data; + unsigned int nr; + int ret; + + if (!jfae) + return -EINVAL; + + nr = (unsigned int)_IOC_NR(cmd); + + switch (nr) { + case JFAE_CMD_GET_ASYNC_EVENT: + ret = cdma_get_async_event(jfae, filp, arg); + break; + default: + dev_err(jfae->cfile->cdev->dev, "nr = %u.\n", nr); + ret = -ENOIOCTLCMD; + break; + } + + return (long)ret; +} + +static int cdma_delete_jfae(struct inode *inode, struct file *filp) +{ + struct cdma_file *cfile; + struct cdma_jfae *jfae; + + if (!filp || !filp->private_data) + return 0; + + jfae = (struct cdma_jfae *)filp->private_data; + cfile = jfae->cfile; + if (!cfile) + return 0; + + if (!mutex_trylock(&cfile->ctx_mutex)) + return -ENOLCK; + jfae->ctx->jfae = NULL; + cdma_uninit_jfe(&jfae->jfe); + kfree(jfae); + filp->private_data = NULL; + mutex_unlock(&cfile->ctx_mutex); + cdma_close_uobj_fd(cfile); + + pr_debug("jfae is release.\n"); + return 0; +} + +static int cdma_jfae_fasync(int fd, struct file *filp, int on) +{ + struct cdma_jfae *jfae = (struct cdma_jfae *)filp->private_data; + int ret; + + if (!jfae) + return -EINVAL; + + spin_lock_irq(&jfae->jfe.lock); + ret = fasync_helper(fd, filp, on, &jfae->jfe.async_queue); + spin_unlock_irq(&jfae->jfe.lock); + + return ret; +} + +const struct file_operations cdma_jfae_fops = { + .owner = THIS_MODULE, + .poll = cdma_jfae_poll, + .unlocked_ioctl = cdma_jfae_ioctl, + .release = cdma_delete_jfae, + .fasync = cdma_jfae_fasync, +}; + +struct cdma_jfae *cdma_alloc_jfae(struct cdma_file *cfile) +{ + struct cdma_jfae *jfae; + struct file *file; + int fd; + + if (!cfile) + return NULL; + + fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); + if (fd < 0) + return NULL; + + jfae = kzalloc(sizeof(*jfae), GFP_KERNEL); + if (!jfae) + goto err_put_unused_fd; + + file = anon_inode_getfile("[jfae]", &cdma_jfae_fops, jfae, + O_RDWR | O_CLOEXEC); + if (IS_ERR(file)) + goto err_free_jfae; + + cdma_init_jfe(&jfae->jfe); + jfae->fd = fd; + jfae->file = file; + jfae->cfile = cfile; + kref_get(&cfile->ref); + fd_install(fd, file); + + return jfae; + +err_free_jfae: + kfree(jfae); +err_put_unused_fd: + put_unused_fd(fd); + + return NULL; +} + +void cdma_free_jfae(struct cdma_jfae *jfae) +{ + if (!jfae) + return; + + fput(jfae->file); + put_unused_fd(jfae->fd); +} + +int cdma_get_jfae(struct cdma_context *ctx) +{ + struct cdma_jfae *jfae; + struct file *file; + + if (!ctx) + return -EINVAL; + + jfae = (struct cdma_jfae *)ctx->jfae; + if (!jfae) + return -EINVAL; + + file = fget(jfae->fd); + if (!file) + return -ENOENT; + + if (file->private_data != jfae) { + fput(file); + return -EBADF; + } + + return 0; +} + +void cdma_init_jfc_event(struct cdma_jfc_event *event, struct cdma_base_jfc *jfc) +{ + event->comp_events_reported = 0; + event->async_events_reported = 0; + INIT_LIST_HEAD(&event->comp_event_list); + INIT_LIST_HEAD(&event->async_event_list); + event->jfc = jfc; +} + +void cdma_release_comp_event(struct cdma_jfce *jfce, struct list_head *event_list) +{ + struct cdma_jfe_event *event, *tmp; + struct cdma_jfe *jfe; + + if (!jfce) + return; + + jfe = &jfce->jfe; + spin_lock_irq(&jfe->lock); + list_for_each_entry_safe(event, tmp, event_list, obj_node) { + list_del(&event->node); + kfree(event); + } + spin_unlock_irq(&jfe->lock); + fput(jfce->file); +} + +void cdma_release_async_event(struct cdma_context *ctx, struct list_head *event_list) +{ + struct cdma_jfe_event *event, *tmp; + struct cdma_jfae *jfae; + struct cdma_jfe *jfe; + + if (!ctx || !ctx->jfae) + return; + + jfae = (struct cdma_jfae *)ctx->jfae; + jfe = &jfae->jfe; + spin_lock_irq(&jfe->lock); + list_for_each_entry_safe(event, tmp, event_list, obj_node) { + list_del(&event->node); + kfree(event); + } + spin_unlock_irq(&jfe->lock); + fput(jfae->file); +} + +void cdma_put_jfae(struct cdma_context *ctx) +{ + struct cdma_jfae *jfae; + + if (!ctx) + return; + + jfae = (struct cdma_jfae *)ctx->jfae; + if (!jfae) + return; + + if (!jfae->file) + return; + + fput(jfae->file); +} diff --git a/drivers/ub/cdma/cdma_event.h b/drivers/ub/cdma/cdma_event.h new file mode 100644 index 0000000000000000000000000000000000000000..4ca14c3c5fcb58d946988da0e3f02613956ea5df --- /dev/null +++ b/drivers/ub/cdma/cdma_event.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_EVENT_H__ +#define __CDMA_EVENT_H__ + +#include +#include +#include +#include +#include +#include +#include "cdma.h" +#include "cdma_context.h" +#include "cdma_types.h" + +#define MAX_EVENT_LIST_SIZE 65535 + +struct cdma_jfe { + spinlock_t lock; + struct list_head event_list; + wait_queue_head_t poll_wait; + struct fasync_struct *async_queue; + uint32_t event_list_count; +}; + +struct cdma_jfae { + int fd; + struct cdma_context *ctx; + struct cdma_file *cfile; + struct file *file; + struct cdma_jfe jfe; +}; + +struct cdma_jfe_event { + struct list_head node; + u32 event_type; + u64 event_data; + struct list_head obj_node; + u32 *counter; +}; + +struct cdma_jfce { + int id; + int fd; + struct cdma_dev *cdev; + struct cdma_file *cfile; + struct file *file; + struct cdma_jfe jfe; +}; + +struct cdma_jfce *cdma_alloc_jfce(struct cdma_file *cfile); + +void cdma_free_jfce(struct cdma_jfce *jfce); + +void cdma_jfs_async_event_cb(struct cdma_event *event, struct cdma_context *ctx); + +void cdma_jfc_async_event_cb(struct cdma_event *event, struct cdma_context *ctx); + +struct cdma_jfae *cdma_alloc_jfae(struct cdma_file *cfile); + +void cdma_free_jfae(struct cdma_jfae *jfae); + +int cdma_get_jfae(struct cdma_context *ctx); + +struct cdma_jfce *cdma_get_jfce_from_id(struct cdma_dev *cdev, int jfce_id); + +void cdma_jfc_comp_event_cb(struct cdma_base_jfc *jfc); + +void cdma_destroy_jfce(struct cdma_jfce *jfce); + +void cdma_init_jfc_event(struct cdma_jfc_event *event, struct cdma_base_jfc *jfc); + +void cdma_release_comp_event(struct cdma_jfce *jfce, struct list_head *event_list); + +void cdma_release_async_event(struct cdma_context *ctx, struct list_head *event_list); + +void cdma_put_jfae(struct cdma_context *ctx); +#endif /* CDMA_EVENT_H */ diff --git a/drivers/ub/cdma/cdma_handle.c b/drivers/ub/cdma/cdma_handle.c new file mode 100644 index 0000000000000000000000000000000000000000..183f802cfdbff7c7b6764dff95418116d2528ca1 --- /dev/null +++ b/drivers/ub/cdma/cdma_handle.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "CDMA: " fmt + +#include "cdma_jfs.h" +#include "cdma_common.h" +#include "cdma_handle.h" + +static int cdma_rw_check(struct cdma_dev *cdev, struct dma_seg *rmt_seg, + struct dma_seg *local_seg) +{ + if (!rmt_seg->len || !local_seg->len) { + dev_err(cdev->dev, "invalid len.\n"); + return -EINVAL; + } + + if (!rmt_seg->sva || !local_seg->sva) { + dev_err(cdev->dev, "invalid address.\n"); + return -EINVAL; + } + + return 0; +} + +static inline void cdma_fill_comm_wr(struct cdma_jfs_wr *wr, + struct cdma_queue *queue) +{ + wr->flag.bs.complete_enable = CDMA_ENABLE_FLAG; + wr->flag.bs.inline_flag = CDMA_DISABLE_FLAG; + wr->flag.bs.fence = CDMA_ENABLE_FLAG; + wr->tpn = queue->tp->tpn; + wr->rmt_eid = queue->cfg.rmt_eid.dw0; + wr->next = NULL; +} + +static inline void cdma_fill_sge(struct cdma_sge_info *rmt_sge, + struct cdma_sge_info *local_sge, + struct dma_seg *rmt_seg, + struct dma_seg *local_seg) +{ + local_sge->addr = local_seg->sva; + local_sge->len = local_seg->len; + local_sge->seg = local_seg; + + rmt_sge->addr = rmt_seg->sva; + rmt_sge->len = rmt_seg->len; + rmt_sge->seg = rmt_seg; +} + +int cdma_write(struct cdma_dev *cdev, struct cdma_queue *queue, + struct dma_seg *local_seg, struct dma_seg *rmt_seg, + struct dma_notify_data *data) +{ + struct cdma_jfs_wr wr = { .opcode = CDMA_WR_OPC_WRITE }; + struct cdma_sge_info rmt_sge, local_sge; + struct cdma_jfs_wr *bad_wr = NULL; + int ret; + + if (cdma_rw_check(cdev, rmt_seg, local_seg)) { + dev_err(cdev->dev, "write param check failed.\n"); + return -EINVAL; + } + + if (data) { + wr.opcode = CDMA_WR_OPC_WRITE_NOTIFY; + wr.rw.notify_addr = data->notify_seg->sva; + wr.rw.notify_data = data->notify_data; + wr.rw.notify_tokenid = data->notify_seg->tid; + wr.rw.notify_tokenvalue = data->notify_seg->token_value; + } + + cdma_fill_comm_wr(&wr, queue); + + cdma_fill_sge(&rmt_sge, &local_sge, rmt_seg, local_seg); + + wr.rw.src.num_sge = 1; + wr.rw.src.sge = &local_sge; + wr.rw.dst.num_sge = 1; + wr.rw.dst.sge = &rmt_sge; + + ret = cdma_post_jfs_wr((struct cdma_jfs *)queue->jfs, &wr, &bad_wr); + if (ret) + dev_err(cdev->dev, "post jfs for write failed, ret = %d.\n", ret); + + return ret; +} + +int cdma_read(struct cdma_dev *cdev, struct cdma_queue *queue, + struct dma_seg *local_seg, struct dma_seg *rmt_seg) +{ + struct cdma_jfs_wr wr = { .opcode = CDMA_WR_OPC_READ }; + struct cdma_sge_info rmt_sge, local_sge; + struct cdma_jfs_wr *bad_wr = NULL; + int ret; + + if (cdma_rw_check(cdev, rmt_seg, local_seg)) { + dev_err(cdev->dev, "read param check failed.\n"); + return -EINVAL; + } + + cdma_fill_comm_wr(&wr, queue); + + cdma_fill_sge(&rmt_sge, &local_sge, rmt_seg, local_seg); + + wr.rw.src.num_sge = 1; + wr.rw.src.sge = &rmt_sge; + wr.rw.dst.num_sge = 1; + wr.rw.dst.sge = &local_sge; + + ret = cdma_post_jfs_wr((struct cdma_jfs *)queue->jfs, &wr, &bad_wr); + if (ret) + dev_err(cdev->dev, "post jfs for read failed, ret = %d.\n", ret); + + return ret; +} + +int cdma_cas(struct cdma_dev *cdev, struct cdma_queue *queue, + struct dma_seg *local_seg, struct dma_seg *rmt_seg, + struct dma_cas_data *data) +{ + struct cdma_jfs_wr wr = { .opcode = CDMA_WR_OPC_CAS }; + struct cdma_sge_info rmt_sge, local_sge; + struct cdma_jfs_wr *bad_wr = NULL; + int ret; + + if (cdma_rw_check(cdev, rmt_seg, local_seg)) { + dev_err(cdev->dev, "cas param check failed.\n"); + return -EINVAL; + } + + cdma_fill_comm_wr(&wr, queue); + + cdma_fill_sge(&rmt_sge, &local_sge, rmt_seg, local_seg); + + wr.cas.src = &local_sge; + wr.cas.dst = &rmt_sge; + + if (local_sge.len <= CDMA_ATOMIC_LEN_8) { + wr.cas.cmp_data = data->compare_data; + wr.cas.swap_data = data->swap_data; + } else { + wr.cas.cmp_addr = data->compare_data; + wr.cas.swap_addr = data->swap_data; + } + + ret = cdma_post_jfs_wr((struct cdma_jfs *)queue->jfs, &wr, &bad_wr); + if (ret) + dev_err(cdev->dev, "post jfs for cas failed, ret = %d.\n", ret); + + return ret; +} + +int cdma_faa(struct cdma_dev *cdev, struct cdma_queue *queue, + struct dma_seg *local_seg, struct dma_seg *rmt_seg, u64 add) +{ + struct cdma_jfs_wr wr = { .opcode = CDMA_WR_OPC_FADD }; + struct cdma_sge_info rmt_sge, local_sge; + struct cdma_jfs_wr *bad_wr = NULL; + int ret; + + if (cdma_rw_check(cdev, rmt_seg, local_seg)) { + dev_err(cdev->dev, "faa param check failed.\n"); + return -EINVAL; + } + + cdma_fill_comm_wr(&wr, queue); + + cdma_fill_sge(&rmt_sge, &local_sge, rmt_seg, local_seg); + + wr.faa.src = &local_sge; + wr.faa.dst = &rmt_sge; + wr.faa.operand = add; + + ret = cdma_post_jfs_wr((struct cdma_jfs *)queue->jfs, &wr, &bad_wr); + if (ret) + dev_err(cdev->dev, "post jfs for faa failed, ret = %d.\n", ret); + + return ret; +} diff --git a/drivers/ub/cdma/cdma_handle.h b/drivers/ub/cdma/cdma_handle.h new file mode 100644 index 0000000000000000000000000000000000000000..00cb8049778ecc7029942c904e2e3ecb7e938ccf --- /dev/null +++ b/drivers/ub/cdma/cdma_handle.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_HANDLE_H__ +#define __CDMA_HANDLE_H__ + +#include "cdma_segment.h" +#include "cdma_queue.h" +#include "cdma.h" + +int cdma_write(struct cdma_dev *cdev, struct cdma_queue *queue, + struct dma_seg *local_seg, struct dma_seg *rmt_seg, + struct dma_notify_data *data); +int cdma_read(struct cdma_dev *cdev, struct cdma_queue *queue, + struct dma_seg *local_seg, struct dma_seg *rmt_seg); +int cdma_cas(struct cdma_dev *cdev, struct cdma_queue *queue, + struct dma_seg *local_seg, struct dma_seg *rmt_seg, + struct dma_cas_data *data); +int cdma_faa(struct cdma_dev *cdev, struct cdma_queue *queue, + struct dma_seg *local_seg, struct dma_seg *rmt_seg, u64 add); + +#endif /* CDMA_HANDLE_H */ diff --git a/drivers/ub/cdma/cdma_ioctl.c b/drivers/ub/cdma/cdma_ioctl.c new file mode 100644 index 0000000000000000000000000000000000000000..0a62e306d6f75b477f8c0c0838886876cf7e328a --- /dev/null +++ b/drivers/ub/cdma/cdma_ioctl.c @@ -0,0 +1,817 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "CDMA: " fmt + +#include + +#include +#include "cdma.h" +#include "cdma_context.h" +#include "cdma_types.h" +#include "cdma_tp.h" +#include "cdma_jfs.h" +#include "cdma_queue.h" +#include "cdma_event.h" +#include "cdma_jfc.h" +#include "cdma_segment.h" +#include "cdma_uobj.h" +#include "cdma_ioctl.h" + +typedef int (*cdma_cmd_handler)(struct cdma_ioctl_hdr *hdr, + struct cdma_file *cfile); + +static void cdma_fill_device_attr(struct cdma_dev *cdev, + struct cdma_device_cap *dev_cap) +{ + dev_cap->max_jfc = cdev->caps.jfc.max_cnt; + dev_cap->max_jfs = cdev->caps.jfs.max_cnt; + dev_cap->max_jfc_depth = cdev->caps.jfc.depth; + dev_cap->max_jfs_depth = cdev->caps.jfs.depth; + dev_cap->max_jfs_sge = cdev->caps.jfs_sge; + dev_cap->max_jfs_rsge = cdev->caps.jfs_rsge; + dev_cap->max_msg_size = cdev->caps.max_msg_len; + dev_cap->trans_mode = cdev->caps.trans_mode; + dev_cap->ceq_cnt = cdev->caps.comp_vector_cnt; +} + +static int cdma_query_dev(struct cdma_ioctl_hdr *hdr, struct cdma_file *cfile) +{ + struct cdma_cmd_query_device_attr_args args = { 0 }; + struct cdma_dev *cdev = cfile->cdev; + unsigned long ret; + + if (!hdr->args_addr || hdr->args_len < sizeof(args)) + return -EINVAL; + + args.out.attr.eid.dw0 = cdev->eid; + args.out.attr.eu_num = cdev->base.attr.eu_num; + memcpy(args.out.attr.eus, cdev->base.attr.eus, + sizeof(struct eu_info) * cdev->base.attr.eu_num); + cdma_fill_device_attr(cdev, &args.out.attr.dev_cap); + + ret = copy_to_user((void __user *)(uintptr_t)hdr->args_addr, &args, + (u32)sizeof(args)); + if (ret) { + dev_err(cdev->dev, "query dev copy to user failed, ret = %lu\n", + ret); + return -EFAULT; + } + + return 0; +} + +static int cdma_create_ucontext(struct cdma_ioctl_hdr *hdr, + struct cdma_file *cfile) +{ + struct cdma_create_context_args args = { 0 }; + struct cdma_dev *cdev = cfile->cdev; + struct cdma_context *ctx; + struct cdma_jfae *jfae; + int ret; + + if (cfile->uctx) { + dev_err(cdev->dev, "create jfae failed, ctx handle = %d.\n", + ctx->handle); + return -EEXIST; + } + + if (!hdr->args_addr || hdr->args_len < sizeof(args)) + return -EINVAL; + + ret = (int)copy_from_user(&args, (void *)hdr->args_addr, + (u32)sizeof(args)); + if (ret) { + dev_err(cdev->dev, "get user data failed, ret = %d.\n", ret); + return ret; + } + + ctx = cdma_alloc_context(cdev, false); + if (IS_ERR(ctx)) + return PTR_ERR(ctx); + + ctx->jfae = cdma_alloc_jfae(cfile); + if (!ctx->jfae) { + dev_err(cdev->dev, "create jfae failed.\n"); + ret = -EFAULT; + goto free_context; + } + + jfae = (struct cdma_jfae *)ctx->jfae; + jfae->ctx = ctx; + args.out.cqe_size = cdev->caps.cqe_size; + args.out.dwqe_enable = + !!(cdev->caps.feature & CDMA_CAP_FEATURE_DIRECT_WQE); + args.out.async_fd = jfae->fd; + cfile->uctx = ctx; + + ret = (int)copy_to_user((void *)hdr->args_addr, &args, + (u32)sizeof(args)); + if (ret) { + dev_err(cdev->dev, "copy ctx to user failed, ret = %d.\n", ret); + goto free_jfae; + } + + return ret; + +free_jfae: + cfile->uctx = NULL; + cdma_free_jfae((struct cdma_jfae *)ctx->jfae); +free_context: + cdma_free_context(cdev, ctx); + + return ret; +} + +static int cdma_delete_ucontext(struct cdma_ioctl_hdr *hdr, + struct cdma_file *cfile) +{ + struct cdma_dev *cdev = cfile->cdev; + + if (!cfile->uctx) { + dev_err(cdev->dev, "cdma context has not been created.\n"); + return -ENOENT; + } + if (!list_empty(&cfile->uctx->queue_list) || + !list_empty(&cfile->uctx->seg_list)) { + dev_err(cdev->dev, + "queue/segment is still in use, ctx handle = %d.\n", + cfile->uctx->handle); + return -EBUSY; + } + + cdma_free_context(cdev, cfile->uctx); + cfile->uctx = NULL; + + return 0; +} + +static int cdma_cmd_create_ctp(struct cdma_ioctl_hdr *hdr, + struct cdma_file *cfile) +{ + struct cdma_cmd_create_ctp_args arg = { 0 }; + struct cdma_dev *cdev = cfile->cdev; + struct cdma_tp_cfg cfg = { 0 }; + struct cdma_base_tp *ctp; + struct cdma_queue *queue; + struct cdma_uobj *uobj; + int ret; + + if (!hdr->args_addr || hdr->args_len < sizeof(arg) || !cfile->uctx) + return -EINVAL; + + ret = (int)copy_from_user(&arg, (void *)hdr->args_addr, + (u32)sizeof(arg)); + if (ret) { + dev_err(&cdev->adev->dev, + "create tp get user data failed, ret = %d.\n", ret); + return -EFAULT; + } + + uobj = cdma_uobj_get(cfile, arg.in.queue_id, UOBJ_TYPE_QUEUE); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, + "create ctp, get queue uobj failed, queue id = %u.\n", + arg.in.queue_id); + return -EINVAL; + } + queue = (struct cdma_queue *)uobj->object; + + uobj = cdma_uobj_create(cfile, UOBJ_TYPE_CTP); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, "create ctp uobj failed.\n"); + return -ENOMEM; + } + + cfg.scna = arg.in.scna; + cfg.dcna = arg.in.dcna; + cfg.seid = arg.in.seid; + cfg.deid = arg.in.deid; + ctp = cdma_create_ctp(cdev, &cfg); + if (!ctp) { + dev_err(&cdev->adev->dev, "create tp failed.\n"); + ret = -EINVAL; + goto delete_obj; + } + uobj->object = ctp; + + arg.out.handle = uobj->id; + arg.out.tpn = ctp->tpn; + ret = (int)copy_to_user((void *)hdr->args_addr, &arg, (u32)sizeof(arg)); + if (ret) { + dev_err(&cdev->adev->dev, + "create tp copy to user data failed, ret = %d.\n", ret); + ret = -EFAULT; + goto delete_ctp; + } + + cdma_set_queue_res(cdev, queue, QUEUE_RES_TP, ctp); + + return 0; + +delete_ctp: + cdma_delete_ctp(cdev, ctp->tp_id); +delete_obj: + cdma_uobj_delete(uobj); + + return ret; +} + +static int cdma_cmd_delete_ctp(struct cdma_ioctl_hdr *hdr, + struct cdma_file *cfile) +{ + struct cdma_cmd_delete_ctp_args arg = { 0 }; + struct cdma_dev *cdev = cfile->cdev; + struct cdma_base_tp *ctp; + struct cdma_queue *queue; + struct cdma_uobj *uobj; + int ret; + + if (!hdr->args_addr || hdr->args_len < sizeof(arg)) + return -EINVAL; + + ret = (int)copy_from_user(&arg, (void *)hdr->args_addr, + (u32)sizeof(arg)); + if (ret) { + dev_err(&cdev->adev->dev, + "delete tp get user data failed, ret = %d.\n", ret); + return -EFAULT; + } + + uobj = cdma_uobj_get(cfile, arg.in.queue_id, UOBJ_TYPE_QUEUE); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, + "delete ctp, get queue uobj failed, queue id = %u.\n", + arg.in.queue_id); + return -EINVAL; + } + queue = uobj->object; + + uobj = cdma_uobj_get(cfile, arg.in.handle, UOBJ_TYPE_CTP); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, "get ctp uobj failed, handle = %llu.\n", + arg.in.handle); + return -EINVAL; + } + ctp = uobj->object; + + cdma_delete_ctp(cdev, ctp->tp_id); + cdma_uobj_delete(uobj); + cdma_set_queue_res(cdev, queue, QUEUE_RES_TP, NULL); + + return ret; +} + +static void cdma_config_jfs(struct cdma_jfs_cfg *cfg, + const struct cdma_cmd_create_jfs_args *arg) +{ + cfg->depth = arg->in.depth; + cfg->flag.value = arg->in.flag; + cfg->eid_index = arg->in.eid_idx; + cfg->max_sge = arg->in.max_sge; + cfg->max_rsge = arg->in.max_rsge; + cfg->rnr_retry = arg->in.rnr_retry; + cfg->err_timeout = arg->in.err_timeout; + cfg->priority = arg->in.priority; + cfg->jfc_id = arg->in.jfc_id; + cfg->rmt_eid = arg->in.rmt_eid; + cfg->pld_token_id = arg->in.pld_token_id; + cfg->tpn = arg->in.tpn; + cfg->queue_id = arg->in.queue_id; + cfg->trans_mode = arg->in.trans_mode; +} + +static int cdma_cmd_create_jfs(struct cdma_ioctl_hdr *hdr, + struct cdma_file *cfile) +{ + struct cdma_cmd_create_jfs_args arg = { 0 }; + struct cdma_dev *cdev = cfile->cdev; + struct cdma_jfs_event *jfs_event; + struct cdma_jfs_cfg cfg = { 0 }; + struct cdma_udata udata = { 0 }; + struct cdma_base_jfs *jfs; + struct cdma_queue *queue; + struct cdma_uobj *uobj; + int ret; + + if (!hdr->args_addr || hdr->args_len != (u32)sizeof(arg) || !cfile->uctx) + return -EINVAL; + + ret = (int)copy_from_user(&arg, (void *)hdr->args_addr, + (u32)sizeof(arg)); + if (ret) { + dev_err(&cdev->adev->dev, + "create jfs get user data failed, ret = %d.\n", ret); + return -EFAULT; + } + + uobj = cdma_uobj_get(cfile, arg.in.queue_id, UOBJ_TYPE_QUEUE); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, + "create jfs, get queue uobj failed, queue id = %u.\n", + arg.in.queue_id); + return -EINVAL; + } + queue = (struct cdma_queue *)uobj->object; + + uobj = cdma_uobj_create(cfile, UOBJ_TYPE_JFS); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, "create jfs uobj failed.\n"); + return -ENOMEM; + } + + udata.uctx = cfile->uctx; + udata.udrv_data = (struct cdma_udrv_priv *)&arg.udata; + arg.in.queue_id = queue->id; + cdma_config_jfs(&cfg, &arg); + + jfs = cdma_create_jfs(cdev, &cfg, &udata); + if (!jfs) { + dev_err(&cdev->adev->dev, "create jfs failed.\n"); + ret = -EFAULT; + goto err_create_jfs; + } + + uobj->object = jfs; + jfs_event = &jfs->jfs_event; + jfs_event->async_events_reported = 0; + INIT_LIST_HEAD(&jfs_event->async_event_list); + + arg.out.id = jfs->id; + arg.out.handle = uobj->id; + arg.out.depth = jfs->cfg.depth; + arg.out.max_sge = jfs->cfg.max_sge; + arg.out.max_rsge = jfs->cfg.max_rsge; + + ret = (int)copy_to_user((void *)hdr->args_addr, &arg, (u32)sizeof(arg)); + if (ret) { + ret = -EFAULT; + dev_err(&cdev->adev->dev, + "create jfs copy to user data failed, ret = %d.\n", + ret); + goto err_copy_to_usr; + } + + cdma_set_queue_res(cdev, queue, QUEUE_RES_JFS, jfs); + + return 0; +err_copy_to_usr: + cdma_delete_jfs(cdev, jfs->id); +err_create_jfs: + cdma_uobj_delete(uobj); + return ret; +} + +static int cdma_cmd_delete_jfs(struct cdma_ioctl_hdr *hdr, + struct cdma_file *cfile) +{ + struct cdma_cmd_delete_jfs_args arg = { 0 }; + struct cdma_dev *cdev = cfile->cdev; + struct cdma_base_jfs *base_jfs; + struct cdma_queue *queue; + struct cdma_uobj *uobj; + int ret; + + if (!hdr->args_addr || hdr->args_len != (u32)sizeof(arg)) + return -EINVAL; + + ret = (int)copy_from_user(&arg, (void *)hdr->args_addr, + (u32)sizeof(arg)); + if (ret) { + dev_err(&cdev->adev->dev, + "delete jfs get user data failed, ret = %d.\n", ret); + return -EFAULT; + } + + uobj = cdma_uobj_get(cfile, arg.in.queue_id, UOBJ_TYPE_QUEUE); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, + "delete jfs, get queue uobj failed, queue id = %u.\n", + arg.in.queue_id); + return -EINVAL; + } + queue = uobj->object; + + uobj = cdma_uobj_get(cfile, arg.in.handle, UOBJ_TYPE_JFS); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, "get jfs uobj failed, handle = %llu.\n", + arg.in.handle); + return -EINVAL; + } + + base_jfs = uobj->object; + ret = cdma_delete_jfs(cdev, base_jfs->id); + if (ret) { + dev_err(&cdev->adev->dev, "delete jfs failed.\n"); + return ret; + } + + cdma_set_queue_res(cdev, queue, QUEUE_RES_JFS, NULL); + cdma_uobj_delete(uobj); + + return 0; +} + +static int cdma_cmd_create_queue(struct cdma_ioctl_hdr *hdr, struct cdma_file *cfile) +{ + struct cdma_cmd_create_queue_args arg = { 0 }; + struct cdma_dev *cdev = cfile->cdev; + struct queue_cfg cfg; + struct cdma_queue *queue; + struct cdma_uobj *uobj; + int ret; + + if (!hdr->args_addr || hdr->args_len != sizeof(arg) || !cfile->uctx) + return -EINVAL; + + ret = (int)copy_from_user(&arg, (void *)hdr->args_addr, + (u32)sizeof(arg)); + if (ret) { + dev_err(cdev->dev, "create queue get user data failed, ret = %d.\n", ret); + return -EFAULT; + } + + cfg = (struct queue_cfg) { + .queue_depth = arg.in.queue_depth, + .dcna = arg.in.dcna, + .priority = arg.in.priority, + .rmt_eid.dw0 = arg.in.rmt_eid, + .user_ctx = arg.in.user_ctx, + .trans_mode = arg.in.trans_mode, + }; + + uobj = cdma_uobj_create(cfile, UOBJ_TYPE_QUEUE); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, "create queue uobj failed.\n"); + return -ENOMEM; + } + + queue = cdma_create_queue(cdev, cfile->uctx, &cfg, 0, false); + if (!queue) { + dev_err(cdev->dev, "create queue failed.\n"); + ret = -EINVAL; + goto err_create_queue; + } + + uobj->object = queue; + arg.out.queue_id = queue->id; + arg.out.handle = uobj->id; + ret = (int)copy_to_user((void *)hdr->args_addr, &arg, (u32)sizeof(arg)); + if (ret) { + dev_err(cdev->dev, "create queue copy to user failed, ret = %d.\n", ret); + ret = -EFAULT; + goto err_copy_to_user; + } + list_add_tail(&queue->list, &cfile->uctx->queue_list); + + return 0; +err_copy_to_user: + cdma_delete_queue(cdev, queue->id); +err_create_queue: + cdma_uobj_delete(uobj); + return ret; +} + +static int cdma_cmd_delete_queue(struct cdma_ioctl_hdr *hdr, struct cdma_file *cfile) +{ + struct cdma_cmd_delete_queue_args arg = { 0 }; + struct cdma_dev *cdev = cfile->cdev; + struct cdma_queue *queue; + struct cdma_uobj *uobj; + int ret; + + if (!hdr->args_addr || hdr->args_len != sizeof(arg)) + return -EINVAL; + + ret = (int)copy_from_user(&arg, (void *)hdr->args_addr, + (u32)sizeof(arg)); + if (ret) { + dev_err(cdev->dev, "delete queue get user data failed, ret = %d.\n", ret); + return -EFAULT; + } + + uobj = cdma_uobj_get(cfile, arg.in.handle, UOBJ_TYPE_QUEUE); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, "get queue uobj failed, handle = %llu.\n", + arg.in.handle); + return -EINVAL; + } + + queue = (struct cdma_queue *)uobj->object; + if (queue->jfc || queue->jfs || queue->tp) { + dev_err(cdev->dev, "jfc/jfs/tp is still in use."); + return -EBUSY; + } + + cdma_uobj_delete(uobj); + list_del(&queue->list); + ret = cdma_delete_queue(cdev, queue->id); + if (ret) + dev_err(cdev->dev, "delete queue failed, ret = %d.\n", ret); + + return ret; +} + +static int cdma_cmd_register_seg(struct cdma_ioctl_hdr *hdr, + struct cdma_file *cfile) +{ + struct cdma_cmd_register_seg_args arg = { 0 }; + struct cdma_dev *cdev = cfile->cdev; + struct dma_seg_cfg cfg = { 0 }; + struct cdma_segment *seg; + struct cdma_uobj *uobj; + int ret; + + if (!hdr->args_addr || hdr->args_len != sizeof(arg) || !cfile->uctx) { + dev_err(cdev->dev, "register seg arg invalid.\n"); + return -EINVAL; + } + + ret = (int)copy_from_user(&arg, (void *)hdr->args_addr, + (u32)sizeof(arg)); + if (ret) { + dev_err(cdev->dev, + "register seg get user data failed, ret = %d.\n", ret); + return -EFAULT; + } + + uobj = cdma_uobj_create(cfile, UOBJ_TYPE_SEGMENT); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, "create seg uobj failed.\n"); + return -ENOMEM; + } + + cfg.sva = arg.in.addr; + cfg.len = arg.in.len; + seg = cdma_register_seg(cdev, &cfg, false); + if (!seg) { + dev_err(cdev->dev, "register seg failed.\n"); + ret = -EINVAL; + goto delete_uobj; + } + seg->ctx = cfile->uctx; + + list_add_tail(&seg->list, &cfile->uctx->seg_list); + arg.out.handle = uobj->id; + uobj->object = seg; + + ret = (int)copy_to_user((void *)hdr->args_addr, &arg, (u32)sizeof(arg)); + if (ret) { + dev_err(cdev->dev, + "register seg copy to user failed, ret = %d.\n", ret); + ret = -EFAULT; + goto free_seg; + } + return 0; + +free_seg: + list_del(&seg->list); + cdma_unregister_seg(cdev, seg); +delete_uobj: + cdma_uobj_delete(uobj); + + return ret; +} + +static int cdma_cmd_unregister_seg(struct cdma_ioctl_hdr *hdr, + struct cdma_file *cfile) +{ + struct cdma_cmd_unregister_seg_args arg = { 0 }; + struct cdma_dev *cdev = cfile->cdev; + struct cdma_segment *seg; + struct cdma_uobj *uobj; + int ret; + + if (!hdr->args_addr || hdr->args_len != sizeof(arg)) { + dev_err(cdev->dev, "unregister seg arg invalid.\n"); + return -EINVAL; + } + + ret = (int)copy_from_user(&arg, (void *)hdr->args_addr, + (u32)sizeof(arg)); + if (ret) { + dev_err(cdev->dev, + "unregister seg get user data failed, ret = %d.\n", + ret); + return -EFAULT; + } + + uobj = cdma_uobj_get(cfile, arg.in.handle, UOBJ_TYPE_SEGMENT); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, "get seg uobj failed.\n"); + return -EINVAL; + } + seg = uobj->object; + list_del(&seg->list); + cdma_unregister_seg(cdev, seg); + cdma_uobj_delete(uobj); + + return ret; +} + +static int cdma_cmd_create_jfc(struct cdma_ioctl_hdr *hdr, + struct cdma_file *cfile) +{ + struct cdma_cmd_create_jfc_args arg = { 0 }; + struct cdma_dev *cdev = cfile->cdev; + struct cdma_jfc_event *jfc_event; + struct cdma_jfc_cfg cfg = { 0 }; + struct cdma_udata udata = { 0 }; + struct cdma_base_jfc *jfc; + struct cdma_queue *queue; + struct cdma_uobj *uobj; + int ret = 0; + + if (!hdr->args_addr || hdr->args_len != (u32)sizeof(arg) || !cfile->uctx) + return -EINVAL; + + ret = (int)copy_from_user(&arg, (void *)hdr->args_addr, + (u32)sizeof(arg)); + if (ret) { + dev_err(cdev->dev, "get user data failed, ret = %d.\n", ret); + return -EFAULT; + } + + uobj = cdma_uobj_get(cfile, arg.in.queue_id, UOBJ_TYPE_QUEUE); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, + "create jfc, get queue uobj failed, queue id = %u.\n", + arg.in.queue_id); + return -EINVAL; + } + queue = (struct cdma_queue *)uobj->object; + + uobj = cdma_uobj_create(cfile, UOBJ_TYPE_JFC); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, "create jfc uobj failed.\n"); + return -ENOMEM; + } + + cfg.depth = arg.in.depth; + cfg.ceqn = arg.in.ceqn; + cfg.queue_id = queue->id; + udata.uctx = cfile->uctx; + udata.udrv_data = (struct cdma_udrv_priv *)&arg.udata; + jfc = cdma_create_jfc(cdev, &cfg, &udata); + if (!jfc) { + dev_err(cdev->dev, "create jfc failed.\n"); + ret = -EFAULT; + goto err_create_jfc; + } + + jfc_event = &jfc->jfc_event; + uobj->object = jfc; + cdma_init_jfc_event(jfc_event, jfc); + + arg.out.id = jfc->id; + arg.out.depth = jfc->jfc_cfg.depth; + arg.out.handle = uobj->id; + jfc_event->jfce = cdma_get_jfce_from_id(cdev, arg.in.jfce_id); + if (!jfc_event->jfce) { + ret = -EFAULT; + goto err_get_jfce; + } + ret = (int)copy_to_user((void *)hdr->args_addr, &arg, (u32)sizeof(arg)); + if (ret != 0) { + dev_err(cdev->dev, "copy jfc to user failed, ret = %d.\n", ret); + ret = -EFAULT; + goto err_copy_to_user; + } else { + cdma_set_queue_res(cdev, queue, QUEUE_RES_JFC, jfc); + } + + return 0; +err_copy_to_user: +err_get_jfce: + cdma_delete_jfc(cdev, jfc->id, NULL); +err_create_jfc: + cdma_uobj_delete(uobj); + return ret; +} + +static int cdma_cmd_delete_jfc(struct cdma_ioctl_hdr *hdr, + struct cdma_file *cfile) +{ + struct cdma_cmd_delete_jfc_args arg = { 0 }; + struct cdma_dev *cdev = cfile->cdev; + struct cdma_base_jfc *base_jfc; + struct cdma_queue *queue; + struct cdma_uobj *uobj; + int ret; + + if (!hdr->args_addr || hdr->args_len != (u32)sizeof(arg)) + return -EINVAL; + + ret = (int)copy_from_user(&arg, (void *)hdr->args_addr, + (u32)sizeof(arg)); + if (ret) { + dev_err(cdev->dev, "get user data failed, ret = %d.\n", ret); + return -EFAULT; + } + + uobj = cdma_uobj_get(cfile, arg.in.queue_id, UOBJ_TYPE_QUEUE); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, + "delete jfc, get queue uobj failed, queue id = %u.\n", + arg.in.queue_id); + return -EINVAL; + } + queue = (struct cdma_queue *)uobj->object; + + uobj = cdma_uobj_get(cfile, arg.in.handle, UOBJ_TYPE_JFC); + if (IS_ERR(uobj)) { + dev_err(cdev->dev, "get jfc uobj failed.\n"); + return -EINVAL; + } + + base_jfc = (struct cdma_base_jfc *)uobj->object; + ret = cdma_delete_jfc(cdev, base_jfc->id, &arg); + if (ret) { + dev_err(cdev->dev, "cdma delete jfc failed, ret = %d.\n", ret); + return -EFAULT; + } + + cdma_set_queue_res(cdev, queue, QUEUE_RES_JFC, NULL); + cdma_uobj_delete(uobj); + + ret = (int)copy_to_user((void *)hdr->args_addr, &arg, (u32)sizeof(arg)); + if (ret) { + dev_err(cdev->dev, + "delete jfc copy to user data failed, ret = %d.\n", + ret); + return -EFAULT; + } + + return 0; +} + +static int cdma_cmd_create_jfce(struct cdma_ioctl_hdr *hdr, + struct cdma_file *cfile) +{ + struct cdma_cmd_create_jfce_args arg = { 0 }; + struct cdma_jfce *jfce; + int ret; + + if (!hdr->args_addr || hdr->args_len != (u32)sizeof(arg)) + return -EINVAL; + + ret = (int)copy_from_user(&arg, (void *)hdr->args_addr, + (u32)sizeof(arg)); + if (ret) + return -EFAULT; + + jfce = cdma_alloc_jfce(cfile); + if (IS_ERR(jfce)) + return PTR_ERR(jfce); + + arg.out.fd = jfce->fd; + arg.out.id = jfce->id; + ret = (int)copy_to_user((void *)hdr->args_addr, &arg, (u32)sizeof(arg)); + if (ret) { + ret = -EFAULT; + goto err_out; + } + + return 0; + +err_out: + cdma_free_jfce(jfce); + + return ret; +} + +static cdma_cmd_handler g_cdma_cmd_handler[CDMA_CMD_MAX] = { + [CDMA_CMD_QUERY_DEV_INFO] = cdma_query_dev, + [CDMA_CMD_CREATE_CTX] = cdma_create_ucontext, + [CDMA_CMD_DELETE_CTX] = cdma_delete_ucontext, + [CDMA_CMD_CREATE_CTP] = cdma_cmd_create_ctp, + [CDMA_CMD_DELETE_CTP] = cdma_cmd_delete_ctp, + [CDMA_CMD_CREATE_JFS] = cdma_cmd_create_jfs, + [CDMA_CMD_DELETE_JFS] = cdma_cmd_delete_jfs, + [CDMA_CMD_REGISTER_SEG] = cdma_cmd_register_seg, + [CDMA_CMD_UNREGISTER_SEG] = cdma_cmd_unregister_seg, + [CDMA_CMD_CREATE_QUEUE] = cdma_cmd_create_queue, + [CDMA_CMD_DELETE_QUEUE] = cdma_cmd_delete_queue, + [CDMA_CMD_CREATE_JFC] = cdma_cmd_create_jfc, + [CDMA_CMD_DELETE_JFC] = cdma_cmd_delete_jfc, + [CDMA_CMD_CREATE_JFCE] = cdma_cmd_create_jfce, +}; + +int cdma_cmd_parse(struct cdma_file *cfile, struct cdma_ioctl_hdr *hdr) +{ + struct cdma_dev *cdev = cfile->cdev; + int ret; + + if (hdr->command >= CDMA_CMD_MAX || !g_cdma_cmd_handler[hdr->command]) { + dev_err(cdev->dev, + "invalid cdma user command or no handler, command = %u\n", + hdr->command); + return -EINVAL; + } + + mutex_lock(&cfile->ctx_mutex); + ret = g_cdma_cmd_handler[hdr->command](hdr, cfile); + mutex_unlock(&cfile->ctx_mutex); + + return ret; +} diff --git a/drivers/ub/cdma/cdma_ioctl.h b/drivers/ub/cdma/cdma_ioctl.h new file mode 100644 index 0000000000000000000000000000000000000000..a5b20c99117e78ccabb4539a68f6e710cd40e93a --- /dev/null +++ b/drivers/ub/cdma/cdma_ioctl.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_IOCTL_H__ +#define __CDMA_IOCTL_H__ + +struct cdma_file; +struct cdma_ioctl_hdr; + +int cdma_cmd_parse(struct cdma_file *cfile, struct cdma_ioctl_hdr *hdr); + +#endif /* _CDMA_IOCTL_H_ */ diff --git a/drivers/ub/cdma/cdma_jfc.c b/drivers/ub/cdma/cdma_jfc.c new file mode 100644 index 0000000000000000000000000000000000000000..0b3611c3d27d3f70db100630024d55e0bc0dfa32 --- /dev/null +++ b/drivers/ub/cdma/cdma_jfc.c @@ -0,0 +1,647 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "CDMA: " fmt + +#include +#include "cdma_cmd.h" +#include "cdma_context.h" +#include "cdma_mbox.h" +#include "cdma_common.h" +#include "cdma_event.h" +#include "cdma_db.h" +#include "cdma_jfs.h" +#include "cdma_jfc.h" + +static int cdma_get_cmd_from_user(struct cdma_create_jfc_ucmd *ucmd, + struct cdma_dev *cdev, + struct cdma_udata *udata, + struct cdma_jfc *jfc, + struct cdma_jfc_cfg *cfg) +{ + struct cdma_context *ctx; + u32 depth = cfg->depth; + int ret; + + if (!udata) { + jfc->arm_sn = 1; + jfc->buf.entry_cnt = depth ? roundup_pow_of_two(depth) : depth; + return 0; + } + + if (!udata->udrv_data || !udata->udrv_data->in_addr || + udata->udrv_data->in_len != (u32)sizeof(*ucmd)) { + dev_err(cdev->dev, "invalid parameter.\n"); + return -EINVAL; + } + + ret = (int)copy_from_user(ucmd, (void *)udata->udrv_data->in_addr, + (u32)sizeof(*ucmd)); + if (ret) { + dev_err(cdev->dev, + "copy udata from user failed, ret = %d.\n", ret); + return -EFAULT; + } + + jfc->mode = ucmd->mode; + jfc->db.db_addr = ucmd->db_addr; + + ctx = udata->uctx; + jfc->base.ctx = ctx; + jfc->tid = ctx->tid; + + if (cdev->caps.cqe_size == CDMA_DEFAULT_CQE_SIZE) + jfc->buf.entry_cnt = ucmd->buf_len >> CDMA_JFC_DEFAULT_CQE_SHIFT; + else + jfc->buf.entry_cnt = ucmd->buf_len >> CDMA_JFC_OTHER_CQE_SHIFT; + + return ret; +} + +static int cdma_check_jfc_cfg(struct cdma_dev *cdev, struct cdma_jfc *jfc, + struct cdma_jfc_cfg *cfg) +{ + if (!jfc->buf.entry_cnt || jfc->buf.entry_cnt > cdev->caps.jfc.depth) { + dev_err(cdev->dev, "invalid jfc depth = %u, cap depth = %u.\n", + jfc->buf.entry_cnt, cdev->caps.jfc.depth); + return -EINVAL; + } + + if (jfc->buf.entry_cnt < CDMA_JFC_DEPTH_MIN) + jfc->buf.entry_cnt = CDMA_JFC_DEPTH_MIN; + + if (cfg->ceqn >= cdev->caps.comp_vector_cnt) { + dev_err(cdev->dev, "invalid ceqn = %u, cap ceq cnt = %u.\n", + cfg->ceqn, cdev->caps.comp_vector_cnt); + return -EINVAL; + } + + return 0; +} + +static void cdma_init_jfc_param(struct cdma_jfc_cfg *cfg, struct cdma_jfc *jfc) +{ + jfc->base.id = jfc->jfcn; + jfc->base.jfc_cfg = *cfg; + jfc->base.jfc_event.jfce = NULL; + jfc->ceqn = cfg->ceqn; +} + +static int cdma_jfc_id_alloc(struct cdma_dev *cdev, struct cdma_jfc *jfc) +{ + struct cdma_table *jfc_tbl = &cdev->jfc_table; + u32 min = jfc_tbl->idr_tbl.min; + u32 max = jfc_tbl->idr_tbl.max; + unsigned long flags; + int id; + + idr_preload(GFP_KERNEL); + spin_lock_irqsave(&jfc_tbl->lock, flags); + id = idr_alloc(&jfc_tbl->idr_tbl.idr, jfc, jfc_tbl->idr_tbl.next, max, + GFP_NOWAIT); + if (id < 0) { + id = idr_alloc(&jfc_tbl->idr_tbl.idr, jfc, min, max, + GFP_NOWAIT); + if (id < 0) + dev_err(cdev->dev, "alloc jfc id failed.\n"); + } + + jfc_tbl->idr_tbl.next = (id >= 0 && id + 1 <= max) ? id + 1 : min; + spin_unlock_irqrestore(&jfc_tbl->lock, flags); + idr_preload_end(); + + jfc->jfcn = id; + + return id; +} + +static void cdma_jfc_id_free(struct cdma_dev *cdev, u32 jfcn) +{ + struct cdma_table *jfc_tbl = &cdev->jfc_table; + unsigned long flags; + + spin_lock_irqsave(&jfc_tbl->lock, flags); + idr_remove(&jfc_tbl->idr_tbl.idr, jfcn); + spin_unlock_irqrestore(&jfc_tbl->lock, flags); +} + +static struct cdma_jfc *cdma_id_find_jfc(struct cdma_dev *cdev, u32 jfcn) +{ + struct cdma_table *jfc_tbl = &cdev->jfc_table; + struct cdma_jfc *jfc; + unsigned long flags; + + spin_lock_irqsave(&jfc_tbl->lock, flags); + jfc = idr_find(&jfc_tbl->idr_tbl.idr, jfcn); + if (!jfc) + dev_err(cdev->dev, "find jfc failed, id = %u.\n", jfcn); + spin_unlock_irqrestore(&jfc_tbl->lock, flags); + + return jfc; +} + +static int cdma_get_jfc_buf(struct cdma_dev *cdev, + struct cdma_create_jfc_ucmd *ucmd, + struct cdma_udata *udata, struct cdma_jfc *jfc) +{ + u32 size; + int ret; + + if (udata) { + jfc->buf.umem = cdma_umem_get(cdev, ucmd->buf_addr, + ucmd->buf_len, false); + if (IS_ERR(jfc->buf.umem)) { + ret = PTR_ERR(jfc->buf.umem); + dev_err(cdev->dev, "get umem failed, ret = %d.\n", + ret); + return ret; + } + jfc->buf.addr = ucmd->buf_addr; + ret = cdma_pin_sw_db(jfc->base.ctx, &jfc->db); + if (ret) + cdma_umem_release(jfc->buf.umem, false); + + return ret; + } + + spin_lock_init(&jfc->lock); + jfc->buf.entry_size = cdev->caps.cqe_size; + jfc->tid = cdev->tid; + size = jfc->buf.entry_size * jfc->buf.entry_cnt; + ret = cdma_k_alloc_buf(cdev, size, &jfc->buf); + if (ret) { + dev_err(cdev->dev, "alloc buffer for jfc failed.\n"); + return ret; + } + + ret = cdma_alloc_sw_db(cdev, &jfc->db); + if (ret) { + dev_err(cdev->dev, "alloc sw db for jfc failed: %u.\n", + jfc->jfcn); + cdma_k_free_buf(cdev, size, &jfc->buf); + } + + return ret; +} + +static void cdma_free_jfc_buf(struct cdma_dev *cdev, struct cdma_jfc *jfc) +{ + u32 size; + + if (!jfc->buf.kva) { + cdma_unpin_sw_db(jfc->base.ctx, &jfc->db); + cdma_unpin_queue_addr(jfc->buf.umem); + } else { + size = jfc->buf.entry_size * jfc->buf.entry_cnt; + cdma_k_free_buf(cdev, size, &jfc->buf); + cdma_free_sw_db(cdev, &jfc->db); + } +} + +static void cdma_construct_jfc_ctx(struct cdma_dev *cdev, + struct cdma_jfc *jfc, + struct cdma_jfc_ctx *ctx) +{ + memset(ctx, 0, sizeof(*ctx)); + + ctx->state = CDMA_JFC_STATE_VALID; + ctx->arm_st = jfc_arm_mode ? CDMA_CTX_NO_ARMED : CDMA_CTX_ALWAYS_ARMED; + ctx->shift = ilog2(jfc->buf.entry_cnt) - CDMA_JFC_DEPTH_SHIFT_BASE; + + if (cdev->caps.cqe_size == CDMA_DEFAULT_CQE_SIZE) + ctx->cqe_size = CDMA_128_CQE_SIZE; + else + ctx->cqe_size = CDMA_64_CQE_SIZE; + + ctx->record_db_en = CDMA_RECORD_EN; + ctx->jfc_type = CDMA_NORMAL_JFC_TYPE; + ctx->cqe_va_l = jfc->buf.addr >> CQE_VA_L_OFFSET; + ctx->cqe_va_h = jfc->buf.addr >> CQE_VA_H_OFFSET; + ctx->cqe_token_id = jfc->tid; + + if (cqe_mode) + ctx->cq_cnt_mode = CDMA_CQE_CNT_MODE_BY_CI_PI_GAP; + else + ctx->cq_cnt_mode = CDMA_CQE_CNT_MODE_BY_COUNT; + + ctx->ceqn = jfc->ceqn; + ctx->record_db_addr_l = jfc->db.db_addr >> CDMA_DB_L_OFFSET; + ctx->record_db_addr_h = jfc->db.db_addr >> CDMA_DB_H_OFFSET; +} + +static int cdma_query_jfc_destroy_done(struct cdma_dev *cdev, uint32_t jfcn) +{ + struct ubase_mbx_attr attr = { 0 }; + struct ubase_cmd_mailbox *mailbox; + struct cdma_jfc_ctx *jfc_ctx; + int ret; + + cdma_fill_mbx_attr(&attr, jfcn, CDMA_CMD_QUERY_JFC_CONTEXT, 0); + mailbox = cdma_mailbox_query_ctx(cdev, &attr); + if (!mailbox) + return -ENOMEM; + + jfc_ctx = mailbox->buf; + ret = jfc_ctx->pi == jfc_ctx->wr_cqe_idx ? 0 : -EAGAIN; + + cdma_free_cmd_mailbox(cdev, mailbox); + + return ret; +} + +static int cdma_destroy_and_flush_jfc(struct cdma_dev *cdev, u32 jfcn) +{ +#define QUERY_MAX_TIMES 5 + u32 wait_times = 0; + int ret; + + ret = cdma_post_destroy_jfc_mbox(cdev, jfcn, CDMA_JFC_STATE_INVALID); + if (ret) { + dev_err(cdev->dev, "post mbox to destroy jfc failed, id: %u.\n", jfcn); + return ret; + } + + while (true) { + if (!cdma_query_jfc_destroy_done(cdev, jfcn)) + return 0; + if (wait_times > QUERY_MAX_TIMES) + break; + msleep(1 << wait_times); + wait_times++; + } + dev_err(cdev->dev, "jfc flush time out, id = %u.\n", jfcn); + + return -ETIMEDOUT; +} + +static inline void *cdma_get_buf_entry(struct cdma_buf *buf, u32 n) +{ + uint32_t entry_index = n & buf->entry_cnt_mask; + + return (char *)buf->kva + (entry_index * buf->entry_size); +} + +static struct cdma_jfc_cqe *cdma_get_next_cqe(struct cdma_jfc *jfc, u32 n) +{ + struct cdma_jfc_cqe *cqe; + u32 valid_owner; + + cqe = (struct cdma_jfc_cqe *)cdma_get_buf_entry(&jfc->buf, n); + valid_owner = (jfc->ci >> jfc->buf.entry_cnt_mask_ilog2) & + CDMA_JFC_DB_VALID_OWNER_M; + if (!(cqe->owner ^ valid_owner)) + return NULL; + + return cqe; +} + +static struct cdma_jetty_queue *cdma_update_jetty_idx(struct cdma_jfc_cqe *cqe) +{ + struct cdma_jetty_queue *queue; + u32 entry_idx; + + entry_idx = cqe->entry_idx; + queue = (struct cdma_jetty_queue *)((u64)cqe->user_data_h << + CDMA_ADDR_SHIFT | cqe->user_data_l); + if (!queue) + return NULL; + + if (!!cqe->fd) + return queue; + + queue->ci += (entry_idx - queue->ci) & (queue->buf.entry_cnt - 1); + + return queue; +} + +static enum jfc_poll_state cdma_get_cr_status(u8 src_status, + u8 substatus, + enum dma_cr_status *dst_status) +{ +struct cdma_cqe_status { + bool is_valid; + enum dma_cr_status cr_status; +}; + + static struct cdma_cqe_status map[CDMA_CQE_STATUS_NUM][CDMA_CQE_SUB_STATUS_NUM] = { + {{true, DMA_CR_SUCCESS}, {false, DMA_CR_SUCCESS}, {false, DMA_CR_SUCCESS}, + {false, DMA_CR_SUCCESS}, {false, DMA_CR_SUCCESS}}, + {{true, DMA_CR_UNSUPPORTED_OPCODE_ERR}, {false, DMA_CR_SUCCESS}, + {false, DMA_CR_SUCCESS}, {false, DMA_CR_SUCCESS}, + {false, DMA_CR_SUCCESS}}, + {{false, DMA_CR_SUCCESS}, {true, DMA_CR_LOC_LEN_ERR}, + {true, DMA_CR_LOC_ACCESS_ERR}, {true, DMA_CR_REM_RESP_LEN_ERR}, + {true, DMA_CR_LOC_DATA_POISON}}, + {{false, DMA_CR_SUCCESS}, {true, DMA_CR_REM_UNSUPPORTED_REQ_ERR}, + {true, DMA_CR_REM_ACCESS_ABORT_ERR}, {false, DMA_CR_SUCCESS}, + {true, DMA_CR_REM_DATA_POISON}}, + {{true, DMA_CR_RNR_RETRY_CNT_EXC_ERR}, {false, DMA_CR_SUCCESS}, + {false, DMA_CR_SUCCESS}, {false, DMA_CR_SUCCESS}, + {false, DMA_CR_SUCCESS}}, + {{true, DMA_CR_ACK_TIMEOUT_ERR}, {false, DMA_CR_SUCCESS}, + {false, DMA_CR_SUCCESS}, {false, DMA_CR_SUCCESS}, + {false, DMA_CR_SUCCESS}}, + {{true, DMA_CR_WR_FLUSH_ERR}, {false, DMA_CR_SUCCESS}, + {false, DMA_CR_SUCCESS}, {false, DMA_CR_SUCCESS}, + {false, DMA_CR_SUCCESS}} + }; + + if ((src_status < CDMA_CQE_STATUS_NUM) && (substatus < CDMA_CQE_SUB_STATUS_NUM) && + map[src_status][substatus].is_valid) { + *dst_status = map[src_status][substatus].cr_status; + return JFC_OK; + } + + return JFC_POLL_ERR; +} + +static enum jfc_poll_state cdma_update_flush_cr(struct cdma_jetty_queue *queue, + struct cdma_jfc_cqe *cqe, + struct dma_cr *cr) +{ + if (cdma_get_cr_status(cqe->status, cqe->substatus, &cr->status)) + return JFC_POLL_ERR; + + if (cqe->fd) { + cr->status = DMA_CR_WR_FLUSH_ERR_DONE; + queue->flush_flag = true; + } else { + queue->ci++; + } + + return JFC_OK; +} + +static enum jfc_poll_state cdma_parse_cqe_for_jfc(struct cdma_dev *cdev, + struct cdma_jfc_cqe *cqe, + struct dma_cr *cr) +{ + struct cdma_jetty_queue *queue; + struct cdma_jfs *jfs; + + queue = cdma_update_jetty_idx(cqe); + if (!queue) { + dev_err(cdev->dev, "update jetty idx failed.\n"); + return JFC_POLL_ERR; + } + + jfs = container_of(queue, struct cdma_jfs, sq); + cr->flag.bs.s_r = cqe->s_r; + cr->flag.bs.jetty = cqe->is_jetty; + cr->completion_len = cqe->byte_cnt; + cr->tpn = cqe->tpn; + cr->local_id = cqe->local_num_h << CDMA_SRC_IDX_SHIFT | cqe->local_num_l; + cr->remote_id = cqe->rmt_idx; + + if (cqe->status) + dev_warn(cdev->dev, "get sq %u cqe status abnormal, ci = %u, pi = %u.\n", + queue->id, queue->ci, queue->pi); + + if (cdma_update_flush_cr(queue, cqe, cr)) { + dev_err(cdev->dev, + "update cr failed, status = %u, substatus = %u.\n", + cqe->status, cqe->substatus); + return JFC_POLL_ERR; + } + + return JFC_OK; +} + +static enum jfc_poll_state cdma_poll_one(struct cdma_dev *cdev, + struct cdma_jfc *jfc, + struct dma_cr *cr) +{ + enum dma_cr_status status; + struct cdma_jfc_cqe *cqe; + + cqe = cdma_get_next_cqe(jfc, jfc->ci); + if (!cqe) + return JFC_EMPTY; + + ++jfc->ci; + /* Ensure that the reading of the event is completed before parsing. */ + rmb(); + + if (cdma_parse_cqe_for_jfc(cdev, cqe, cr)) + return JFC_POLL_ERR; + + status = cr->status; + if (status == DMA_CR_WR_FLUSH_ERR_DONE || status == DMA_CR_WR_SUSPEND_DONE) { + dev_info(cdev->dev, "poll cr flush/suspend done, jfc id = %u, status = %u.\n", + jfc->jfcn, status); + return JFC_EMPTY; + } + + return JFC_OK; +} + +static void cdma_release_jfc_event(struct cdma_jfc *jfc) +{ + cdma_release_comp_event(jfc->base.jfc_event.jfce, + &jfc->base.jfc_event.comp_event_list); + cdma_release_async_event(jfc->base.ctx, + &jfc->base.jfc_event.async_event_list); +} + +static int cdma_post_create_jfc_mbox(struct cdma_dev *cdev, struct cdma_jfc *jfc) +{ + struct ubase_mbx_attr attr = { 0 }; + struct cdma_jfc_ctx ctx = { 0 }; + + cdma_construct_jfc_ctx(cdev, jfc, &ctx); + cdma_fill_mbx_attr(&attr, jfc->jfcn, CDMA_CMD_CREATE_JFC_CONTEXT, 0); + + return cdma_post_mailbox_ctx(cdev, (void *)&ctx, sizeof(ctx), &attr); +} + +int cdma_post_destroy_jfc_mbox(struct cdma_dev *cdev, u32 jfcn, + enum cdma_jfc_state state) +{ + struct ubase_mbx_attr attr = { 0 }; + struct cdma_jfc_ctx ctx = { 0 }; + + ctx.state = state; + cdma_fill_mbx_attr(&attr, jfcn, CDMA_CMD_DESTROY_JFC_CONTEXT, 0); + + return cdma_post_mailbox_ctx(cdev, (void *)&ctx, sizeof(ctx), &attr); +} + +struct cdma_base_jfc *cdma_create_jfc(struct cdma_dev *cdev, + struct cdma_jfc_cfg *cfg, + struct cdma_udata *udata) +{ + struct cdma_create_jfc_ucmd ucmd = { 0 }; + struct cdma_jfc *jfc; + int ret; + + jfc = kzalloc(sizeof(*jfc), GFP_KERNEL); + if (!jfc) + return NULL; + + ret = cdma_get_cmd_from_user(&ucmd, cdev, udata, jfc, cfg); + if (ret) + goto err_get_cmd; + + ret = cdma_check_jfc_cfg(cdev, jfc, cfg); + if (ret) + goto err_check_cfg; + + ret = cdma_jfc_id_alloc(cdev, jfc); + if (ret < 0) + goto err_alloc_jfc_id; + + cdma_init_jfc_param(cfg, jfc); + ret = cdma_get_jfc_buf(cdev, &ucmd, udata, jfc); + if (ret) + goto err_get_jfc_buf; + + if (udata) { + ret = cdma_get_jfae(jfc->base.ctx); + if (ret) + goto err_get_jfae; + } + + ret = cdma_post_create_jfc_mbox(cdev, jfc); + if (ret) + goto err_alloc_cqc; + + refcount_set(&jfc->event_refcount, 1); + init_completion(&jfc->event_comp); + jfc->base.jfae_handler = cdma_jfc_async_event_cb; + jfc->base.jfce_handler = cdma_jfc_comp_event_cb; + jfc->base.dev = cdev; + + dev_dbg(cdev->dev, "create jfc id = %u, queue id = %u.\n", + jfc->jfcn, cfg->queue_id); + + return &jfc->base; + +err_alloc_cqc: + if (udata) + cdma_put_jfae(jfc->base.ctx); +err_get_jfae: + cdma_free_jfc_buf(cdev, jfc); +err_get_jfc_buf: + cdma_jfc_id_free(cdev, jfc->jfcn); +err_alloc_jfc_id: +err_check_cfg: +err_get_cmd: + kfree(jfc); + return NULL; +} + +int cdma_delete_jfc(struct cdma_dev *cdev, u32 jfcn, + struct cdma_cmd_delete_jfc_args *arg) +{ + struct cdma_jfc_event *jfc_event; + struct cdma_jfc *jfc; + int ret; + + if (!cdev) + return -EINVAL; + + if (jfcn >= cdev->caps.jfc.max_cnt + cdev->caps.jfc.start_idx || + jfcn < cdev->caps.jfc.start_idx) { + dev_err(cdev->dev, + "jfc id invalid, jfcn = %u, start_idx = %u, max_cnt = %u.\n", + jfcn, cdev->caps.jfc.start_idx, + cdev->caps.jfc.max_cnt); + return -EINVAL; + } + + jfc = cdma_id_find_jfc(cdev, jfcn); + if (!jfc) { + dev_err(cdev->dev, "find jfc failed, jfcn = %u.\n", jfcn); + return -EINVAL; + } + + if (!(jfc->base.ctx && jfc->base.ctx->invalid)) { + ret = cdma_destroy_and_flush_jfc(cdev, jfc->jfcn); + if (ret) + dev_err(cdev->dev, "jfc delete failed, jfcn = %u.\n", jfcn); + } + + if (refcount_dec_and_test(&jfc->event_refcount)) + complete(&jfc->event_comp); + wait_for_completion(&jfc->event_comp); + + cdma_free_jfc_buf(cdev, jfc); + cdma_jfc_id_free(cdev, jfc->jfcn); + if (arg) { + jfc_event = &jfc->base.jfc_event; + arg->out.comp_events_reported = jfc_event->comp_events_reported; + arg->out.async_events_reported = jfc_event->async_events_reported; + } + + pr_debug("Leave %s, jfcn: %u.\n", __func__, jfc->jfcn); + + cdma_release_jfc_event(jfc); + kfree(jfc); + + return 0; +} + +int cdma_jfc_completion(struct notifier_block *nb, unsigned long jfcn, + void *data) +{ + struct auxiliary_device *adev = (struct auxiliary_device *)data; + struct cdma_base_jfc *base_jfc; + struct cdma_table *jfc_tbl; + struct cdma_dev *cdev; + struct cdma_jfc *jfc; + unsigned long flags; + + if (!adev) + return -EINVAL; + + cdev = get_cdma_dev(adev); + jfc_tbl = &cdev->jfc_table; + spin_lock_irqsave(&jfc_tbl->lock, flags); + jfc = idr_find(&jfc_tbl->idr_tbl.idr, jfcn); + if (!jfc) { + dev_warn(cdev->dev, "can not find jfc, jfcn = %lu.\n", jfcn); + spin_unlock_irqrestore(&jfc_tbl->lock, flags); + return -EINVAL; + } + + ++jfc->arm_sn; + base_jfc = &jfc->base; + if (base_jfc->jfce_handler) { + refcount_inc(&jfc->event_refcount); + spin_unlock_irqrestore(&jfc_tbl->lock, flags); + base_jfc->jfce_handler(base_jfc); + if (refcount_dec_and_test(&jfc->event_refcount)) + complete(&jfc->event_comp); + } else { + spin_unlock_irqrestore(&jfc_tbl->lock, flags); + } + + return 0; +} + +/* thanks to drivers/infiniband/hw/bnxt_re/ib_verbs.c */ +int cdma_poll_jfc(struct cdma_base_jfc *base_jfc, int cr_cnt, + struct dma_cr *cr) +{ + struct cdma_jfc *jfc = to_cdma_jfc(base_jfc); + enum jfc_poll_state err = JFC_OK; + int npolled = 0; + + jfc->buf.entry_cnt_mask = jfc->buf.entry_cnt - 1; + jfc->buf.entry_cnt_mask_ilog2 = ilog2(jfc->buf.entry_cnt); + + spin_lock(&jfc->lock); + + for (npolled = 0; npolled < cr_cnt; ++npolled) { + err = cdma_poll_one(base_jfc->dev, jfc, cr + npolled); + if (err != JFC_OK) + break; + } + + if (npolled) + *jfc->db.db_record = jfc->ci & (u32)CDMA_JFC_DB_CI_IDX_M; + + spin_unlock(&jfc->lock); + + return err == JFC_POLL_ERR ? -CDMA_INTER_ERR : npolled; +} diff --git a/drivers/ub/cdma/cdma_jfc.h b/drivers/ub/cdma/cdma_jfc.h new file mode 100644 index 0000000000000000000000000000000000000000..7f512150e50c9fcc779f3dd52b1572b68e3380bb --- /dev/null +++ b/drivers/ub/cdma/cdma_jfc.h @@ -0,0 +1,191 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_JFC_H__ +#define __CDMA_JFC_H__ + +#include "cdma_types.h" +#include "cdma_db.h" + +#define CDMA_JFC_DEPTH_MIN 64 +#define CDMA_JFC_DEPTH_SHIFT_BASE 6 +#define CDMA_JFC_DEFAULT_CQE_SHIFT 7 +#define CDMA_JFC_OTHER_CQE_SHIFT 6 + +#define CDMA_DB_L_OFFSET 6 +#define CDMA_DB_H_OFFSET 38 + +#define CQE_VA_L_OFFSET 12 +#define CQE_VA_H_OFFSET 32 + +#define CDMA_IMM_DATA_SHIFT 32 + +enum cdma_record_db { + CDMA_NO_RECORD_EN, + CDMA_RECORD_EN +}; + +enum cdma_jfc_state { + CDMA_JFC_STATE_INVALID, + CDMA_JFC_STATE_VALID, + CDMA_JFC_STATE_ERROR +}; + +enum cdma_armed_jfc { + CDMA_CTX_NO_ARMED, + CDMA_CTX_ALWAYS_ARMED, + CDMA_CTX_REG_NEXT_CEQE, + CDMA_CTX_REG_NEXT_SOLICITED_CEQE +}; + +enum cdma_jfc_type { + CDMA_NORMAL_JFC_TYPE, + CDMA_RAW_JFC_TYPE +}; + +enum cdma_cq_cnt_mode { + CDMA_CQE_CNT_MODE_BY_COUNT, + CDMA_CQE_CNT_MODE_BY_CI_PI_GAP +}; + +struct cdma_jfc { + struct cdma_base_jfc base; + u32 jfcn; + u32 ceqn; + u32 tid; + struct cdma_buf buf; + struct cdma_sw_db db; + u32 ci; + u32 arm_sn; + spinlock_t lock; + refcount_t event_refcount; + struct completion event_comp; + u32 mode; +}; + +struct cdma_jfc_ctx { + /* DW0 */ + u32 state : 2; + u32 arm_st : 2; + u32 shift : 4; + u32 cqe_size : 1; + u32 record_db_en : 1; + u32 jfc_type : 1; + u32 inline_en : 1; + u32 cqe_va_l : 20; + /* DW1 */ + u32 cqe_va_h; + /* DW2 */ + u32 cqe_token_id : 20; + u32 cq_cnt_mode : 1; + u32 rsv0 : 3; + u32 ceqn : 8; + /* DW3 */ + u32 cqe_token_value : 24; + u32 rsv1 : 8; + /* DW4 */ + u32 pi : 22; + u32 cqe_coalesce_cnt : 10; + /* DW5 */ + u32 ci : 22; + u32 cqe_coalesce_period : 3; + u32 rsv2 : 7; + /* DW6 */ + u32 record_db_addr_l; + /* DW7 */ + u32 record_db_addr_h : 26; + u32 rsv3 : 6; + /* DW8 */ + u32 push_usi_en : 1; + u32 push_cqe_en : 1; + u32 token_en : 1; + u32 rsv4 : 9; + u32 tpn : 20; + /* DW9 ~ DW12 */ + u32 rmt_eid[4]; + /* DW13 */ + u32 seid_idx : 10; + u32 rmt_token_id : 20; + u32 rsv5 : 2; + /* DW14 */ + u32 remote_token_value; + /* DW15 */ + u32 int_vector : 16; + u32 stars_en : 1; + u32 rsv6 : 15; + /* DW16 */ + u32 poll : 1; + u32 cqe_report_timer : 24; + u32 se : 1; + u32 arm_sn : 2; + u32 rsv7 : 4; + /* DW17 */ + u32 se_cqe_idx : 24; + u32 rsv8 : 8; + /* DW18 */ + u32 wr_cqe_idx : 22; + u32 rsv9 : 10; + /* DW19 */ + u32 cqe_cnt : 24; + u32 rsv10 : 8; + /* DW20 ~ DW31 */ + u32 rsv11[12]; +}; + +struct cdma_jfc_cqe { + /* DW0 */ + u32 s_r : 1; + u32 is_jetty : 1; + u32 owner : 1; + u32 inline_en : 1; + u32 opcode : 3; + u32 fd : 1; + u32 rsv : 8; + u32 substatus : 8; + u32 status : 8; + /* DW1 */ + u32 entry_idx : 16; + u32 local_num_l : 16; + /* DW2 */ + u32 local_num_h : 4; + u32 rmt_idx : 20; + u32 rsv1 : 8; + /* DW3 */ + u32 tpn : 24; + u32 rsv2 : 8; + /* DW4 */ + u32 byte_cnt; + /* DW5 ~ DW6 */ + u32 user_data_l; + u32 user_data_h; + /* DW7 ~ DW10 */ + u32 rmt_eid[4]; + /* DW11 ~ DW12 */ + u32 data_l; + u32 data_h; + /* DW13 ~ DW15 */ + u32 inline_data[3]; +}; + +static inline struct cdma_jfc *to_cdma_jfc(struct cdma_base_jfc *base_jfc) +{ + return container_of(base_jfc, struct cdma_jfc, base); +} + +int cdma_post_destroy_jfc_mbox(struct cdma_dev *cdev, u32 jfcn, + enum cdma_jfc_state state); + +struct cdma_base_jfc *cdma_create_jfc(struct cdma_dev *cdev, + struct cdma_jfc_cfg *cfg, + struct cdma_udata *udata); + +int cdma_delete_jfc(struct cdma_dev *cdev, u32 jfcn, + struct cdma_cmd_delete_jfc_args *arg); + +int cdma_jfc_completion(struct notifier_block *nb, unsigned long jfcn, + void *data); + +int cdma_poll_jfc(struct cdma_base_jfc *base_jfc, int cr_cnt, + struct dma_cr *cr); + +#endif /* CDMA_JFC_H */ diff --git a/drivers/ub/cdma/cdma_jfs.c b/drivers/ub/cdma/cdma_jfs.c new file mode 100644 index 0000000000000000000000000000000000000000..8a62e2a2fd6bab2a05cab9aa77d1ce40597eed58 --- /dev/null +++ b/drivers/ub/cdma/cdma_jfs.c @@ -0,0 +1,1048 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define pr_fmt(fmt) "CDMA: " fmt +#define dev_fmt pr_fmt + +#include +#include +#include +#include +#include +#include "cdma_cmd.h" +#include "cdma_common.h" +#include "cdma_mbox.h" +#include "cdma_event.h" +#include "cdma_context.h" +#include "cdma_jfs.h" + +static int cdma_get_user_jfs_cmd(struct cdma_dev *cdev, struct cdma_jfs *jfs, + struct cdma_udata *udata, + struct cdma_create_jfs_ucmd *ucmd) +{ + struct cdma_context *ctx; + int ret; + + if (!udata) { + jfs->jfs_addr = (uintptr_t)&jfs->sq; + jfs->is_kernel = true; + return 0; + } + + if (!udata->udrv_data || !udata->udrv_data->in_addr || + udata->udrv_data->in_len != (u32)sizeof(*ucmd)) { + dev_err(cdev->dev, "invalid parameter.\n"); + return -EINVAL; + } + + ret = (int)copy_from_user(ucmd, (void *)udata->udrv_data->in_addr, + (u32)sizeof(*ucmd)); + if (ret) { + dev_err(cdev->dev, + "copy jfs udata failed, ret = %d.\n", ret); + return -EFAULT; + } + + if (!ucmd->jetty_addr || !ucmd->buf_len || !ucmd->buf_addr) { + dev_err(cdev->dev, "user cmd param is invalid.\n"); + return -EINVAL; + } + + ctx = udata->uctx; + jfs->base_jfs.ctx = ctx; + jfs->sq.tid = ctx->tid; + jfs->jfs_addr = ucmd->jetty_addr; + jfs->sq.id = ucmd->jfs_id; + jfs->queue_id = ucmd->queue_id; + jfs->sq.non_pin = ucmd->non_pin; + + return 0; +} + +static int cdma_alloc_jfs_id(struct cdma_dev *cdev, struct cdma_jfs *jfs) +{ + struct cdma_idr *idr_tbl = &cdev->jfs_table.idr_tbl; + u32 max = idr_tbl->max; + u32 min = idr_tbl->min; + int id; + + idr_preload(GFP_KERNEL); + spin_lock(&cdev->jfs_table.lock); + id = idr_alloc(&idr_tbl->idr, jfs, idr_tbl->next, max, GFP_NOWAIT); + if (id < 0) { + id = idr_alloc(&idr_tbl->idr, jfs, min, max, GFP_NOWAIT); + if (id < 0) + dev_err(cdev->dev, "alloc cdma jfs id failed.\n"); + } + + idr_tbl->next = (id >= 0 && id + 1 <= max) ? id + 1 : min; + spin_unlock(&cdev->jfs_table.lock); + idr_preload_end(); + + return id; +} + +static inline u32 cdma_sq_cal_wqebb_num(u32 sqe_ctl_len, u32 sge_num) +{ + return (sqe_ctl_len + (sge_num - 1) * CDMA_JFS_SGE_SIZE) / CDMA_JFS_WQEBB_SIZE + 1; +} + +static inline void cdma_set_kernel_db(struct cdma_dev *cdev, + struct cdma_jetty_queue *queue) +{ + queue->dwqe_addr = + cdev->k_db_base + JETTY_DSQE_OFFSET + PAGE_SIZE * queue->id; + queue->db_addr = queue->dwqe_addr + CDMA_DOORBELL_OFFSET; +} + +static int cdma_get_sq_buf(struct cdma_dev *cdev, struct cdma_jetty_queue *sq, + struct cdma_jfs_cfg *jfs_cfg, + struct cdma_create_jfs_ucmd *ucmd, bool is_kernel) +{ + u32 wqe_bb_depth; + u32 sqe_bb_cnt; + int ret = 0; + u32 size; + + if (!is_kernel) { + ret = cdma_pin_queue_addr(cdev, ucmd->buf_addr, + ucmd->buf_len, &sq->buf); + if (ret) { + dev_err(cdev->dev, + "pin jfs queue addr failed, ret = %d.\n", + ret); + return ret; + } + + sq->buf.entry_cnt = ucmd->buf_len >> WQE_BB_SIZE_SHIFT; + sq->sqe_bb_cnt = ucmd->sqe_bb_cnt; + if (sq->sqe_bb_cnt > MAX_WQEBB_NUM) + sq->sqe_bb_cnt = MAX_WQEBB_NUM; + } else { + spin_lock_init(&sq->lock); + sq->tid = cdev->tid; + sq->max_sge_num = jfs_cfg->max_sge; + sqe_bb_cnt = + cdma_sq_cal_wqebb_num(SQE_WRITE_NOTIFY_CTL_LEN, + jfs_cfg->max_sge); + if (sqe_bb_cnt > MAX_WQEBB_NUM) + sqe_bb_cnt = MAX_WQEBB_NUM; + sq->sqe_bb_cnt = sqe_bb_cnt; + + wqe_bb_depth = roundup_pow_of_two(sqe_bb_cnt * jfs_cfg->depth); + sq->buf.entry_size = CDMA_JFS_WQEBB_SIZE; + size = ALIGN(wqe_bb_depth * sq->buf.entry_size, CDMA_HW_PAGE_SIZE); + sq->buf.entry_cnt = size >> WQE_BB_SIZE_SHIFT; + + ret = cdma_k_alloc_buf(cdev, size, &sq->buf); + if (ret) { + dev_err(cdev->dev, + "alloc jfs (%u) sq buf failed, size = %u.\n", + sq->id, size); + return ret; + } + + cdma_set_kernel_db(cdev, sq); + sq->kva_curr = sq->buf.kva; + } + + return ret; +} + +static void cdma_init_jfsc(struct cdma_dev *cdev, struct cdma_jfs_cfg *cfg, + struct cdma_jfs *jfs, void *mb_buf) +{ + struct cdma_jfs_ctx *ctx = mb_buf; + + ctx->state = CDMA_JETTY_READY; + ctx->sl = cdev->sl[cfg->priority % cdev->sl_num]; + ctx->jfs_mode = CDMA_JFS_MODE; + ctx->type = (cfg->trans_mode == CDMA_JETTY_ROL) ? CDMA_JETTY_ROL : CDMA_JETTY_ROI; + ctx->sqe_base_addr_l = (jfs->sq.buf.addr >> SQE_VA_L_OFFSET) & + (u32)SQE_VA_L_VALID_BIT; + ctx->sqe_base_addr_h = (jfs->sq.buf.addr >> SQE_VA_H_OFFSET) & + (u32)SQE_VA_H_VALID_BIT; + ctx->sqe_token_id_l = jfs->sq.tid & (u32)SQE_TOKEN_ID_L_MASK; + ctx->sqe_token_id_h = (jfs->sq.tid >> SQE_TOKEN_ID_H_OFFSET) & + (u32)SQE_TOKEN_ID_H_MASK; + ctx->sqe_bb_shift = ilog2(roundup_pow_of_two(jfs->sq.buf.entry_cnt)); + ctx->tx_jfcn = cfg->jfc_id; + ctx->ta_timeout = cfg->err_timeout; + ctx->rnr_retry_num = cfg->rnr_retry; + ctx->user_data_l = jfs->jfs_addr; + ctx->user_data_h = jfs->jfs_addr >> CDMA_USER_DATA_H_OFFSET; + ctx->seid_idx = cfg->eid_index; + ctx->err_mode = cfg->flag.bs.error_suspend; + ctx->cmp_odr = cfg->flag.bs.outorder_comp; + ctx->avail_sgmt_ost = AVAIL_SGMT_OST_INIT; + ctx->sqe_pld_tokenid = jfs->sq.tid & (u32)SQE_PLD_TOKEN_ID_MASK; + ctx->next_send_ssn = get_random_u16(); + ctx->next_rcv_ssn = ctx->next_send_ssn; + + ctx->sqe_pos = cfg->sqe_pos; + ctx->sqe_pld_pos = cfg->pld_pos; + ctx->rmt_eid = cfg->rmt_eid; + ctx->rmt_tokenid = cfg->pld_token_id; + ctx->tpn = cfg->tpn; +} + +static inline void cdma_reset_jfs_queue(struct cdma_jetty_queue *sq) +{ + sq->kva_curr = sq->buf.kva; + sq->pi = 0; + sq->ci = 0; + sq->flush_flag = false; +} + +static int cdma_create_hw_jfs_ctx(struct cdma_dev *cdev, struct cdma_jfs *jfs, + struct cdma_jfs_cfg *cfg) +{ + struct ubase_mbx_attr attr = { 0 }; + struct cdma_jfs_ctx ctx = { 0 }; + int ret; + + cdma_init_jfsc(cdev, cfg, jfs, &ctx); + cdma_fill_mbx_attr(&attr, jfs->sq.id, CDMA_CMD_CREATE_JFS_CONTEXT, 0); + ret = cdma_post_mailbox_ctx(cdev, &ctx, sizeof(ctx), &attr); + if (ret) { + dev_err(cdev->dev, "upgrade jfs ctx failed, ret = %d.\n", ret); + return ret; + } + + cdma_reset_jfs_queue(&jfs->sq); + + return 0; +} + +static void cdma_free_sq_buf(struct cdma_dev *cdev, struct cdma_jetty_queue *sq) +{ + u32 size; + + if (sq->buf.kva) { + size = sq->buf.entry_cnt * sq->buf.entry_size; + cdma_k_free_buf(cdev, size, &sq->buf); + } else { + cdma_unpin_queue_addr(sq->buf.umem); + sq->buf.umem = NULL; + } +} + +static void cdma_set_query_flush_time(struct cdma_jetty_queue *sq, + u8 err_timeout) +{ + static u32 time[] = { + CDMA_TA_TIMEOUT_128MS, + CDMA_TA_TIMEOUT_1000MS, + CDMA_TA_TIMEOUT_8000MS, + CDMA_TA_TIMEOUT_64000MS, + }; + static u8 time_index_max = ARRAY_SIZE(time) - 1; + + if (err_timeout > time_index_max) + err_timeout = time_index_max; + + sq->ta_tmo = time[err_timeout]; +} + +static inline void cdma_free_jfs_id(struct cdma_dev *cdev, u32 id) +{ + spin_lock(&cdev->jfs_table.lock); + idr_remove(&cdev->jfs_table.idr_tbl.idr, id); + spin_unlock(&cdev->jfs_table.lock); +} + +static int cdma_verify_jfs_cfg(struct cdma_dev *cdev, struct cdma_jfs_cfg *cfg) +{ + if (!cfg->depth || cfg->depth > cdev->caps.jfs.depth) { + dev_err(cdev->dev, + "jfs param is invalid, depth = %u, max_depth = %u.\n", + cfg->depth, cdev->caps.jfs.depth); + return -EINVAL; + } + + return 0; +} + +struct cdma_base_jfs *cdma_create_jfs(struct cdma_dev *cdev, + struct cdma_jfs_cfg *cfg, + struct cdma_udata *udata) +{ + struct cdma_create_jfs_ucmd ucmd = { 0 }; + struct cdma_jfs *jfs; + int ret; + + if (cdma_verify_jfs_cfg(cdev, cfg)) + return NULL; + + jfs = kzalloc(sizeof(*jfs), GFP_KERNEL); + if (!jfs) + return NULL; + + ret = cdma_get_user_jfs_cmd(cdev, jfs, udata, &ucmd); + if (ret) + goto err_alloc_jfsn; + + ret = cdma_alloc_jfs_id(cdev, jfs); + if (ret < 0) + goto err_alloc_jfsn; + + jfs->id = ret; + jfs->sq.id = ret; + jfs->base_jfs.id = jfs->sq.id; + jfs->base_jfs.cfg = *cfg; + jfs->dev = cdev; + jfs->queue_id = cfg->queue_id; + + ret = cdma_get_sq_buf(cdev, &jfs->sq, cfg, &ucmd, jfs->is_kernel); + if (ret) + goto err_get_jfs_buf; + + if (udata) { + ret = cdma_get_jfae(jfs->base_jfs.ctx); + if (ret) + goto err_get_jfae; + } + + ret = cdma_create_hw_jfs_ctx(cdev, jfs, cfg); + if (ret) + goto err_create_hw_jfsc; + + cdma_set_query_flush_time(&jfs->sq, cfg->err_timeout); + refcount_set(&jfs->ae_ref_cnt, 1); + init_completion(&jfs->ae_comp); + jfs->sq.state = CDMA_JETTY_READY; + jfs->base_jfs.jfae_handler = cdma_jfs_async_event_cb; + jfs->base_jfs.dev = cdev; + + dev_dbg(cdev->dev, + "create jfs id = %u, queue id = %u, depth = %u, priority = %u, jfc id = %u.\n", + jfs->id, jfs->queue_id, cfg->depth, cfg->priority, cfg->jfc_id); + + return &jfs->base_jfs; + +err_create_hw_jfsc: + if (udata) + cdma_put_jfae(jfs->base_jfs.ctx); +err_get_jfae: + cdma_free_sq_buf(cdev, &jfs->sq); +err_get_jfs_buf: + cdma_free_jfs_id(cdev, jfs->sq.id); +err_alloc_jfsn: + kfree(jfs); + + return NULL; +} + +static int cdma_set_jfs_state(struct cdma_dev *cdev, u32 jfs_id, + enum cdma_jetty_state state) +{ + struct cdma_jfs_ctx ctx[SZ_2] = { 0 }; + struct ubase_mbx_attr attr = { 0 }; + struct cdma_jfs_ctx *ctx_mask; + + ctx_mask = (struct cdma_jfs_ctx *)((char *)ctx + SZ_128); + memset(ctx_mask, 0xff, sizeof(*ctx_mask)); + ctx->state = state; + ctx_mask->state = 0; + + cdma_fill_mbx_attr(&attr, jfs_id, CDMA_CMD_MODIFY_JFS_CONTEXT, 0); + + return cdma_post_mailbox_ctx(cdev, (void *)ctx, sizeof(ctx), &attr); +} + +static int cdma_query_jfs_ctx(struct cdma_dev *cdev, + struct cdma_jfs_ctx *jfs_ctx, + u32 jfs_id) +{ + struct ubase_mbx_attr attr = { 0 }; + struct ubase_cmd_mailbox *mailbox; + + cdma_fill_mbx_attr(&attr, jfs_id, CDMA_CMD_QUERY_JFS_CONTEXT, 0); + mailbox = cdma_mailbox_query_ctx(cdev, &attr); + if (!mailbox) + return -ENOMEM; + memcpy((void *)jfs_ctx, mailbox->buf, sizeof(*jfs_ctx)); + + cdma_free_cmd_mailbox(cdev, mailbox); + + return 0; +} + +static int cdma_destroy_hw_jfs_ctx(struct cdma_dev *cdev, u32 jfs_id) +{ + struct ubase_mbx_attr attr = { 0 }; + int ret; + + cdma_fill_mbx_attr(&attr, jfs_id, CDMA_CMD_DESTROY_JFS_CONTEXT, 0); + ret = cdma_post_mailbox_ctx(cdev, NULL, 0, &attr); + if (ret) + dev_err(cdev->dev, + "post mailbox destroy jfs ctx failed, ret = %d.\n", ret); + + return ret; +} + +static bool cdma_wait_timeout(u32 *sum_times, u32 times, u32 ta_timeout) +{ + u32 wait_time; + + if (*sum_times > ta_timeout) + return true; + + wait_time = 1 << times; + msleep(wait_time); + *sum_times += wait_time; + + return false; +} + +static bool cdma_query_jfs_fd(struct cdma_dev *cdev, + struct cdma_jetty_queue *sq) +{ + struct cdma_jfs_ctx ctx = { 0 }; + u16 rcv_send_diff = 0; + u32 sum_times = 0; + u32 times = 0; + + while (true) { + if (cdma_query_jfs_ctx(cdev, &ctx, sq->id)) + return false; + + if (ctx.flush_cqe_done) + return true; + + if (cdma_wait_timeout(&sum_times, times, sq->ta_tmo)) { + dev_warn(cdev->dev, + "ta timeout, id = %u. PI = %u, CI = %u, next_send_ssn = %u next_rcv_ssn = %u state = %u.\n", + sq->id, ctx.pi, ctx.ci, ctx.next_send_ssn, + ctx.next_rcv_ssn, ctx.state); + break; + } + + times++; + } + + /* In the flip scenario, ctx.next_rcv_ssn - ctx.next_send_ssn value is less than 512. */ + rcv_send_diff = ctx.next_rcv_ssn - ctx.next_send_ssn; + if (ctx.flush_ssn_vld && rcv_send_diff < CDMA_RCV_SEND_MAX_DIFF) + return true; + + dev_err(cdev->dev, "query jfs flush ssn error, id = %u", sq->id); + + return false; +} + +int cdma_modify_jfs_precondition(struct cdma_dev *cdev, + struct cdma_jetty_queue *sq) +{ + struct cdma_jfs_ctx ctx = { 0 }; + u16 rcv_send_diff = 0; + u32 sum_times = 0; + u32 times = 0; + + while (true) { + if (cdma_query_jfs_ctx(cdev, &ctx, sq->id)) { + dev_err(cdev->dev, "query jfs ctx failed, id = %u.\n", + sq->id); + return -ENOMEM; + } + + rcv_send_diff = ctx.next_rcv_ssn - ctx.next_send_ssn; + if ((ctx.pi == ctx.ci) && (rcv_send_diff < CDMA_RCV_SEND_MAX_DIFF) && + (ctx.state == CDMA_JETTY_READY)) + break; + + if ((rcv_send_diff < CDMA_RCV_SEND_MAX_DIFF) && + (ctx.state == CDMA_JETTY_ERROR)) + break; + + if (cdma_wait_timeout(&sum_times, times, sq->ta_tmo)) { + dev_warn(cdev->dev, + "ta timeout, id = %u. PI = %u, CI = %u, next_send_ssn = %u next_rcv_ssn = %u state = %u.\n", + sq->id, ctx.pi, ctx.ci, ctx.next_send_ssn, + ctx.next_rcv_ssn, ctx.state); + break; + } + times++; + } + + return 0; +} + +static bool cdma_destroy_jfs_precondition(struct cdma_dev *cdev, + struct cdma_jetty_queue *sq) +{ +#define CDMA_DESTROY_JETTY_DELAY_TIME 100U + + if ((sq->state == CDMA_JETTY_READY) || + (sq->state == CDMA_JETTY_SUSPENDED)) { + if (cdma_modify_jfs_precondition(cdev, sq)) + return false; + + if (cdma_set_jfs_state(cdev, sq->id, CDMA_JETTY_ERROR)) { + dev_err(cdev->dev, "modify jfs state to error failed, id = %u.\n", + sq->id); + return false; + } + + sq->state = CDMA_JETTY_ERROR; + dev_dbg(cdev->dev, "set jfs %u status finished.\n", sq->id); + } + + if (!cdma_query_jfs_fd(cdev, sq)) + return false; + + udelay(CDMA_DESTROY_JETTY_DELAY_TIME); + + return true; +} + +static int cdma_modify_and_destroy_jfs(struct cdma_dev *cdev, + struct cdma_jetty_queue *sq) +{ + int ret = 0; + + if (!cdma_destroy_jfs_precondition(cdev, sq)) + return -EINVAL; + + if (sq->state != CDMA_JETTY_RESET) + ret = cdma_destroy_hw_jfs_ctx(cdev, sq->id); + + return ret; +} + +static inline void cdma_release_jfs_event(struct cdma_jfs *jfs) +{ + cdma_release_async_event(jfs->base_jfs.ctx, + &jfs->base_jfs.jfs_event.async_event_list); +} + +int cdma_delete_jfs(struct cdma_dev *cdev, u32 jfs_id) +{ + struct cdma_jfs *jfs; + int ret; + + if (jfs_id >= cdev->caps.jfs.start_idx + cdev->caps.jfs.max_cnt) { + dev_info(cdev->dev, + "jfs id invalid, jfs_id = %u, start_idx = %u, max_cnt = %u.\n", + jfs_id, cdev->caps.jfs.start_idx, + cdev->caps.jfs.max_cnt); + return -EINVAL; + } + + spin_lock(&cdev->jfs_table.lock); + jfs = idr_find(&cdev->jfs_table.idr_tbl.idr, jfs_id); + spin_unlock(&cdev->jfs_table.lock); + if (!jfs) { + dev_err(cdev->dev, "get jfs from table failed, id = %u.\n", jfs_id); + return -EINVAL; + } + + if (!(jfs->base_jfs.ctx && jfs->base_jfs.ctx->invalid)) { + ret = cdma_modify_and_destroy_jfs(cdev, &jfs->sq); + if (ret) + dev_err(cdev->dev, "jfs delete failed, id = %u.\n", jfs->id); + } + + if (refcount_dec_and_test(&jfs->ae_ref_cnt)) + complete(&jfs->ae_comp); + wait_for_completion(&jfs->ae_comp); + + cdma_free_sq_buf(cdev, &jfs->sq); + + cdma_free_jfs_id(cdev, jfs_id); + + pr_debug("Leave %s, jfsn: %u.\n", __func__, jfs_id); + + cdma_release_jfs_event(jfs); + + kfree(jfs); + + return 0; +} + +static u8 cdma_get_jfs_opcode(enum cdma_wr_opcode opcode) +{ + switch (opcode) { + case CDMA_WR_OPC_WRITE: + return CDMA_OPC_WRITE; + case CDMA_WR_OPC_WRITE_NOTIFY: + return CDMA_OPC_WRITE_WITH_NOTIFY; + case CDMA_WR_OPC_READ: + return CDMA_OPC_READ; + case CDMA_WR_OPC_CAS: + return CDMA_OPC_CAS; + case CDMA_WR_OPC_FADD: + return CDMA_OPC_FAA; + default: + return CDMA_OPC_INVALID; + } +} + +static inline u32 cdma_get_normal_sge_num(u8 opcode, struct cdma_sqe_ctl *tmp_sq) +{ + switch (opcode) { + case CDMA_OPC_CAS: + case CDMA_OPC_FAA: + return CDMA_ATOMIC_SGE_NUM_ATOMIC; + default: + return tmp_sq->sge_num; + } +} + +static bool cdma_k_check_sge_num(u8 opcode, struct cdma_jetty_queue *sq, + struct cdma_jfs_wr *wr) +{ + switch (opcode) { + case CDMA_OPC_CAS: + case CDMA_OPC_FAA: + return sq->max_sge_num == 0; + case CDMA_OPC_READ: + return wr->rw.dst.num_sge > sq->max_sge_num; + case CDMA_OPC_WRITE_WITH_NOTIFY: + return wr->rw.src.num_sge > CDMA_JFS_MAX_SGE_NOTIFY || + wr->rw.src.num_sge > sq->max_sge_num; + default: + return wr->rw.src.num_sge > sq->max_sge_num; + } +} + +static int cdma_fill_sw_sge(struct cdma_sqe_ctl *sqe_ctl, + struct cdma_jfs_wr *wr, + struct cdma_normal_sge *sge) +{ + struct cdma_sge_info *sge_info; + u32 sge_num = 0; + u32 num_sge; + u32 i; + + switch (wr->opcode) { + case CDMA_WR_OPC_WRITE: + case CDMA_WR_OPC_WRITE_NOTIFY: + sge_info = wr->rw.src.sge; + num_sge = wr->rw.src.num_sge; + break; + default: + return -EINVAL; + } + + for (i = 0; i < num_sge; i++) { + if (sge_info[i].len == 0) + continue; + sge->va = sge_info[i].addr; + sge->length = sge_info[i].len; + sge->token_id = sge_info[i].seg->tid; + sge++; + sge_num++; + } + sqe_ctl->sge_num = sge_num; + + return 0; +} + +static inline u32 cdma_get_ctl_len(u8 opcode) +{ + if (opcode == CDMA_OPC_WRITE_WITH_NOTIFY) + return SQE_WRITE_NOTIFY_CTL_LEN; + + return SQE_NORMAL_CTL_LEN; +} + +static int cdma_k_fill_write_sqe(struct cdma_dev *cdev, + struct cdma_sqe_ctl *sqe_ctl, + struct cdma_jfs_wr *wr) +{ + struct cdma_token_info *token_info; + struct cdma_sge_info *sge_info; + struct cdma_normal_sge *sge; + u32 ctrl_len; + + ctrl_len = cdma_get_ctl_len(sqe_ctl->opcode); + sge = (struct cdma_normal_sge *)((void *)sqe_ctl + ctrl_len); + + if (cdma_fill_sw_sge(sqe_ctl, wr, sge)) + return -EINVAL; + + sge_info = wr->rw.dst.sge; + + sqe_ctl->toid = sge_info[0].seg->tid; + sqe_ctl->token_en = sge_info[0].seg->token_value_valid; + sqe_ctl->rmt_token_value = sge_info[0].seg->token_value; + sqe_ctl->target_hint = wr->rw.target_hint; + sqe_ctl->rmt_addr_l_or_token_id = + sge_info[0].addr & (u32)SQE_CTL_RMA_ADDR_BIT; + sqe_ctl->rmt_addr_h_or_token_value = + (sge_info[0].addr >> (u32)SQE_CTL_RMA_ADDR_OFFSET) & + (u32)SQE_CTL_RMA_ADDR_BIT; + + if (sqe_ctl->opcode == CDMA_OPC_WRITE_WITH_NOTIFY) { + token_info = (struct cdma_token_info *) + ((void *)sqe_ctl + SQE_NOTIFY_TOKEN_ID_FIELD); + token_info->token_id = wr->rw.notify_tokenid; + token_info->token_value = wr->rw.notify_tokenvalue; + + memcpy((void *)sqe_ctl + SQE_NOTIFY_ADDR_FIELD, + &wr->rw.notify_addr, sizeof(u64)); + memcpy((void *)sqe_ctl + SQE_ATOMIC_DATA_FIELD, + &wr->rw.notify_data, sizeof(u64)); + } + + return 0; +} + +static int cdma_k_fill_read_sqe(struct cdma_dev *cdev, + struct cdma_sqe_ctl *sqe_ctl, + struct cdma_jfs_wr *wr) +{ + struct cdma_sge_info *sge_info; + struct cdma_normal_sge *sge; + u32 sge_num = 0; + u32 num; + + sge = (struct cdma_normal_sge *)(sqe_ctl + 1); + sge_info = wr->rw.dst.sge; + + for (num = 0; num < wr->rw.dst.num_sge; num++) { + if (!sge_info[num].len) + continue; + sge->va = sge_info[num].addr; + sge->length = sge_info[num].len; + sge->token_id = sge_info[num].seg->tid; + sge++; + sge_num++; + } + + sge_info = wr->rw.src.sge; + sqe_ctl->sge_num = sge_num; + sqe_ctl->toid = sge_info[0].seg->tid; + sqe_ctl->token_en = sge_info[0].seg->token_value_valid; + sqe_ctl->rmt_token_value = sge_info[0].seg->token_value; + sqe_ctl->rmt_addr_l_or_token_id = + sge_info[0].addr & (u32)SQE_CTL_RMA_ADDR_BIT; + sqe_ctl->rmt_addr_h_or_token_value = + (sge_info[0].addr >> (u32)SQE_CTL_RMA_ADDR_OFFSET) & + (u32)SQE_CTL_RMA_ADDR_BIT; + + return 0; +} + +static bool cdma_check_atomic_len(u32 len, u8 opcode) +{ + switch (len) { + case CDMA_ATOMIC_LEN_4: + case CDMA_ATOMIC_LEN_8: + return true; + case CDMA_ATOMIC_LEN_16: + if (opcode == CDMA_WR_OPC_CAS) + return true; + return false; + default: + return false; + } +} + +static int cdma_k_fill_cas_sqe(struct cdma_dev *cdev, + struct cdma_sqe_ctl *sqe_ctl, + struct cdma_jfs_wr *wr) +{ + struct cdma_sge_info *sge_info; + struct cdma_normal_sge *sge; + + sge_info = wr->cas.src; + if (!cdma_check_atomic_len(sge_info->len, wr->opcode)) { + dev_err(cdev->dev, "cdma cas sge len invalid, len = %u.\n", + sge_info->len); + return -EINVAL; + } + + sge = (struct cdma_normal_sge *)(sqe_ctl + 1); + sge->va = sge_info->addr; + sge->length = sge_info->len; + sge->token_id = sge_info->seg->tid; + + sge_info = wr->cas.dst; + sqe_ctl->sge_num = CDMA_ATOMIC_SGE_NUM; + sqe_ctl->toid = sge_info->seg->tid; + sqe_ctl->token_en = sge_info->seg->token_value_valid; + sqe_ctl->rmt_token_value = sge_info->seg->token_value; + sqe_ctl->rmt_addr_l_or_token_id = sge_info->addr & + (u32)SQE_CTL_RMA_ADDR_BIT; + sqe_ctl->rmt_addr_h_or_token_value = + (sge_info->addr >> (u32)SQE_CTL_RMA_ADDR_OFFSET) & + (u32)SQE_CTL_RMA_ADDR_BIT; + + if (sge->length <= CDMA_ATOMIC_LEN_8) { + memcpy((void *)sqe_ctl + SQE_ATOMIC_DATA_FIELD, + &wr->cas.swap_data, sge->length); + memcpy((void *)sqe_ctl + SQE_ATOMIC_DATA_FIELD + sge->length, + &wr->cas.cmp_data, sge->length); + } else { + memcpy((void *)sqe_ctl + SQE_ATOMIC_DATA_FIELD, + (char *)wr->cas.swap_addr, sge->length); + memcpy((void *)sqe_ctl + SQE_ATOMIC_DATA_FIELD + sge->length, + (char *)wr->cas.cmp_addr, sge->length); + } + + return 0; +} + +static int cdma_k_fill_faa_sqe(struct cdma_dev *cdev, + struct cdma_sqe_ctl *sqe_ctl, + struct cdma_jfs_wr *wr) +{ + struct cdma_sge_info *sge_info; + struct cdma_normal_sge *sge; + + sge_info = wr->faa.src; + if (!cdma_check_atomic_len(sge_info->len, wr->opcode)) { + dev_err(cdev->dev, "cdma faa sge len invalid, len = %u.\n", + sge_info->len); + return -EINVAL; + } + + sge = (struct cdma_normal_sge *)(sqe_ctl + 1); + sge->va = sge_info->addr; + sge->length = sge_info->len; + sge->token_id = sge_info->seg->tid; + + sge_info = wr->faa.dst; + sqe_ctl->sge_num = CDMA_ATOMIC_SGE_NUM; + sqe_ctl->toid = sge_info->seg->tid; + sqe_ctl->token_en = sge_info->seg->token_value_valid; + sqe_ctl->rmt_token_value = sge_info->seg->token_value; + sqe_ctl->rmt_addr_l_or_token_id = sge_info->addr & + (u32)SQE_CTL_RMA_ADDR_BIT; + sqe_ctl->rmt_addr_h_or_token_value = + (sge_info->addr >> (u32)SQE_CTL_RMA_ADDR_OFFSET) & + (u32)SQE_CTL_RMA_ADDR_BIT; + + if (sge->length <= CDMA_ATOMIC_LEN_8) + memcpy((void *)sqe_ctl + SQE_ATOMIC_DATA_FIELD, + &wr->faa.operand, sge->length); + else + memcpy((void *)sqe_ctl + SQE_ATOMIC_DATA_FIELD, + (void *)wr->faa.operand_addr, sge->length); + + return 0; +} + +static int cdma_fill_normal_sge(struct cdma_dev *cdev, + struct cdma_sqe_ctl *sqe_ctl, + struct cdma_jfs_wr *wr) +{ + switch (wr->opcode) { + case CDMA_WR_OPC_WRITE: + case CDMA_WR_OPC_WRITE_NOTIFY: + return cdma_k_fill_write_sqe(cdev, sqe_ctl, wr); + case CDMA_WR_OPC_READ: + return cdma_k_fill_read_sqe(cdev, sqe_ctl, wr); + case CDMA_WR_OPC_CAS: + return cdma_k_fill_cas_sqe(cdev, sqe_ctl, wr); + case CDMA_WR_OPC_FADD: + return cdma_k_fill_faa_sqe(cdev, sqe_ctl, wr); + default: + dev_err(cdev->dev, "cdma wr opcode invalid, opcode = %u.\n", + (u8)wr->opcode); + return -EINVAL; + } +} + +static int cdma_set_sqe(struct cdma_dev *cdev, struct cdma_sqe_ctl *sqe_ctl, + struct cdma_jfs_wr *wr, struct cdma_jetty_queue *sq, + u8 opcode) +{ + int ret; + + sqe_ctl->cqe = wr->flag.bs.complete_enable; + sqe_ctl->owner = (sq->pi & sq->buf.entry_cnt) == 0 ? 1 : 0; + sqe_ctl->opcode = opcode; + sqe_ctl->tpn = wr->tpn; + sqe_ctl->place_odr = wr->flag.bs.place_order; + sqe_ctl->fence = wr->flag.bs.fence; + sqe_ctl->comp_order = wr->flag.bs.comp_order; + sqe_ctl->se = wr->flag.bs.solicited_enable; + sqe_ctl->inline_en = 0; + memcpy(sqe_ctl->rmt_eid, &wr->rmt_eid, sizeof(wr->rmt_eid)); + + ret = cdma_fill_normal_sge(cdev, sqe_ctl, wr); + if (ret) + dev_err(cdev->dev, + "cdma fill normal sge failed, wr opcode = %u.\n", + (u8)wr->opcode); + + return ret; +} + +static u32 cdma_cal_wqebb_num(struct cdma_jfs_wr *wr, u8 opcode, + struct cdma_sqe_ctl *tmp_sq) +{ + u32 normal_sge_num; + u32 sqe_ctl_len; + u32 wqebb_cnt; + + sqe_ctl_len = cdma_get_ctl_len(opcode); + + normal_sge_num = cdma_get_normal_sge_num(opcode, tmp_sq); + wqebb_cnt = cdma_sq_cal_wqebb_num(sqe_ctl_len, normal_sge_num); + + return wqebb_cnt; +} + +static inline bool to_check_sq_overflow(struct cdma_jetty_queue *sq, + u32 wqebb_cnt) +{ + return (sq->pi - sq->ci + wqebb_cnt) > sq->buf.entry_cnt; +} + +static int cdma_copy_to_sq(struct cdma_jetty_queue *sq, u32 wqebb_cnt, + struct cdma_jfs_wqebb *tmp_sq) +{ + u32 remain = sq->buf.entry_cnt - (sq->pi & (sq->buf.entry_cnt - 1)); + u32 tail_cnt; + u32 head_cnt; + + if (to_check_sq_overflow(sq, wqebb_cnt)) + return -ENOMEM; + + tail_cnt = remain > wqebb_cnt ? wqebb_cnt : remain; + head_cnt = wqebb_cnt - tail_cnt; + + memcpy(sq->kva_curr, tmp_sq, tail_cnt * sizeof(*tmp_sq)); + if (head_cnt) + memcpy(sq->buf.kva, tmp_sq + tail_cnt, + head_cnt * sizeof(*tmp_sq)); + + return 0; +} + +static void *cdma_k_update_ptr(u32 total_size, u32 wqebb_size, u8 *base_addr, + u8 *curr_addr) +{ + u8 *end_addr; + + end_addr = base_addr + total_size; + curr_addr = ((curr_addr + wqebb_size) < end_addr) ? + (curr_addr + wqebb_size) : + base_addr + (curr_addr + wqebb_size - end_addr); + + return curr_addr; +} + +static int cdma_post_one_wr(struct cdma_jetty_queue *sq, struct cdma_jfs_wr *wr, + struct cdma_dev *cdev, + struct cdma_sqe_ctl **dwqe_addr, u8 *dwqe_enable) +{ + struct cdma_jfs_wqebb tmp_sq[MAX_WQEBB_NUM] = { 0 }; + u32 wqebb_cnt; + u8 opcode; + int ret; + + opcode = cdma_get_jfs_opcode(wr->opcode); + if (opcode == CDMA_OPC_INVALID) { + dev_err(cdev->dev, "cdma invalid opcode = %u.\n", wr->opcode); + return -EINVAL; + } + + if (cdma_k_check_sge_num(opcode, sq, wr)) { + dev_err(cdev->dev, "cdma sge num invalid, opcode = %u.\n", + opcode); + return -EINVAL; + } + + ret = cdma_set_sqe(cdev, (struct cdma_sqe_ctl *)tmp_sq, wr, sq, opcode); + if (ret) + return ret; + + wqebb_cnt = + cdma_cal_wqebb_num(wr, opcode, (struct cdma_sqe_ctl *)tmp_sq); + if (wqebb_cnt == 1 && + !!(cdev->caps.feature & CDMA_CAP_FEATURE_DIRECT_WQE)) + *dwqe_enable = 1; + + ret = cdma_copy_to_sq(sq, wqebb_cnt, tmp_sq); + if (ret) { + dev_err(cdev->dev, "cdma jfs overflow, wqebb_cnt = %u.\n", + wqebb_cnt); + return ret; + } + + *dwqe_addr = sq->kva_curr; + + sq->kva_curr = cdma_k_update_ptr(sq->buf.entry_cnt * sq->buf.entry_size, + wqebb_cnt * sq->buf.entry_size, + (u8 *)sq->buf.kva, (u8 *)sq->kva_curr); + + sq->pi += wqebb_cnt; + + return 0; +} + +static void cdma_write_dsqe(struct cdma_jetty_queue *sq, + struct cdma_sqe_ctl *ctrl) +{ +#define DWQE_SIZE 8 + int i; + + ctrl->sqe_bb_idx = sq->pi; + for (i = 0; i < DWQE_SIZE; i++) + writeq_relaxed(*((u64 *)ctrl + i), (u64 *)sq->dwqe_addr + i); +} + +static inline void cdma_k_update_sq_db(struct cdma_jetty_queue *sq) +{ + u32 *db_addr = (u32 *)sq->db_addr; + *db_addr = sq->pi; +} + +/* thanks to drivers/infiniband/hw/bnxt_re/ib_verbs.c */ +static int cdma_post_sq_wr(struct cdma_dev *cdev, struct cdma_jetty_queue *sq, + struct cdma_jfs_wr *wr, struct cdma_jfs_wr **bad_wr) +{ + struct cdma_sqe_ctl *dwqe_addr; + struct cdma_jfs_wr *it; + u8 dwqe_enable = 0; + int wr_cnt = 0; + int ret = 0; + + spin_lock(&sq->lock); + + for (it = wr; it != NULL; it = it->next) { + ret = cdma_post_one_wr(sq, it, cdev, &dwqe_addr, &dwqe_enable); + if (ret) { + dev_err(cdev->dev, "cdma post one wr failed.\n"); + *bad_wr = it; + goto post_wr; + } + wr_cnt++; + } + +post_wr: + if (wr_cnt) { + if (cdev->status != CDMA_SUSPEND) { + /* Ensure the order of write memory operations */ + wmb(); + if (wr_cnt == 1 && dwqe_enable && (sq->pi - sq->ci == 1)) + cdma_write_dsqe(sq, dwqe_addr); + else + cdma_k_update_sq_db(sq); + } + } + + spin_unlock(&sq->lock); + + return ret; +} + +int cdma_post_jfs_wr(struct cdma_jfs *jfs, struct cdma_jfs_wr *wr, + struct cdma_jfs_wr **bad_wr) +{ + struct cdma_dev *cdev = jfs->dev; + int ret; + + ret = cdma_post_sq_wr(cdev, &jfs->sq, wr, bad_wr); + if (ret) + dev_err(cdev->dev, + "cdma post jfs wr failed, sq_id = %u.\n", jfs->sq.id); + + return ret; +} diff --git a/drivers/ub/cdma/cdma_jfs.h b/drivers/ub/cdma/cdma_jfs.h new file mode 100644 index 0000000000000000000000000000000000000000..3d0391b03d977a4ac261b5641fa9dcf3cf1222e0 --- /dev/null +++ b/drivers/ub/cdma/cdma_jfs.h @@ -0,0 +1,329 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_JFS_H__ +#define __CDMA_JFS_H__ + +#include "cdma_common.h" +#include "cdma_types.h" +#include "cdma_segment.h" + +#define MAX_WQEBB_NUM 4 +#define CDMA_SQE_RMT_EID_SIZE 4 +#define CDMA_JFS_WQEBB_SIZE 64 +#define SQE_NORMAL_CTL_LEN 48 +#define CDMA_JFS_MAX_SGE_NOTIFY 11 +#define CDMA_JFS_SGE_SIZE 16 +#define SQE_WRITE_NOTIFY_CTL_LEN 80 +#define CDMA_ATOMIC_SGE_NUM 1 +#define CDMA_ATOMIC_SGE_NUM_ATOMIC 2 +#define SQE_CTL_RMA_ADDR_OFFSET 32 +#define SQE_CTL_RMA_ADDR_BIT GENMASK(31, 0) +#define SQE_NOTIFY_TOKEN_ID_FIELD 48 +#define SQE_NOTIFY_ADDR_FIELD 56 +#define SQE_ATOMIC_DATA_FIELD 64 + +#define CDMA_TA_TIMEOUT_128MS 128 +#define CDMA_TA_TIMEOUT_1000MS 1000 +#define CDMA_TA_TIMEOUT_8000MS 8000 +#define CDMA_TA_TIMEOUT_64000MS 64000 + +#define CDMA_RCV_SEND_MAX_DIFF 512U + +struct cdma_jfs_wqebb { + u32 value[16]; +}; + +struct cdma_token_info { + u32 token_id : 20; + u32 rsv : 12; + u32 token_value; +}; + +struct cdma_sqe_ctl { + /* DW0 */ + u32 sqe_bb_idx : 16; + u32 place_odr : 2; + u32 comp_order : 1; + u32 fence : 1; + u32 se : 1; + u32 cqe : 1; + u32 inline_en : 1; + u32 rsv : 5; + u32 token_en : 1; + u32 rmt_jetty_type : 2; + u32 owner : 1; + /* DW1 */ + u32 target_hint : 8; + u32 opcode : 8; + u32 rsv1 : 6; + u32 inline_msg_len : 10; + /* DW2 */ + u32 tpn : 24; + u32 sge_num : 8; + /* DW3 */ + u32 toid : 20; + u32 rsv2 : 12; + /* DW4~7 */ + u32 rmt_eid[CDMA_SQE_RMT_EID_SIZE]; + /* DW8 */ + u32 rmt_token_value; + /* DW9~11 */ + u32 rsv3; + u32 rmt_addr_l_or_token_id; + u32 rmt_addr_h_or_token_value; +}; + +union cdma_jfs_wr_flag { + struct { + /* 0: There is no order with other WR. + * 1: relax order. + * 2: strong order. + * 3: reserve. + */ + u32 place_order : 2; + /* 0: There is no completion order with other WR + * 1: Completion order with previous WR. + */ + u32 comp_order : 1; + /* 0: There is no fence. + * 1: Fence with previous read and atomic WR + */ + u32 fence : 1; + /* 0: not solicited. + * 1: solicited. It will trigger an event + * on remote side + */ + u32 solicited_enable : 1; + /* 0: Do not notify local process + * after the task is complete. + * 1: Notify local process + * after the task is completed. + */ + u32 complete_enable : 1; + /* 0: No inline. + * 1: Inline data. + */ + u32 inline_flag : 1; + u32 reserved : 25; + } bs; + u32 value; +}; + +struct cdma_sge_info { + u64 addr; + u32 len; + struct dma_seg *seg; +}; + +struct cdma_normal_sge { + u32 length; + u32 token_id; + u64 va; +}; + +struct cdma_sg { + struct cdma_sge_info *sge; + u32 num_sge; +}; + +struct cdma_rw_wr { + struct cdma_sg src; + struct cdma_sg dst; + u8 target_hint; /* hint of jetty in a target jetty group */ + u64 notify_data; /* notify data or immeditate data in host byte order */ + u64 notify_addr; + u32 notify_tokenid; + u32 notify_tokenvalue; +}; + +struct cdma_cas_wr { + struct cdma_sge_info *dst; /* len in the sge is the length of CAS + * operation, only support 8/16/32B + */ + struct cdma_sge_info *src; /* local address for destination original + * value written back + */ + union { + u64 cmp_data; /* when the len is 8B, it indicates the compare value. */ + u64 cmp_addr; /* when the len is 16/32B, it indicates the data address. */ + }; + union { + /* if destination value is the same as cmp_data, + * destination value will be change to swap_data. + */ + u64 swap_data; + u64 swap_addr; + }; +}; + +struct cdma_faa_wr { + struct cdma_sge_info *dst; /* len in the sge is the length of FAA + * operation, only support 4/8B + */ + struct cdma_sge_info *src; /* local address for destination original + * value written back + */ + union { + u64 operand; /* Addend */ + u64 operand_addr; + }; +}; + +struct cdma_jfs_wr { + enum cdma_wr_opcode opcode; + union cdma_jfs_wr_flag flag; + u32 tpn; + u32 rmt_eid; + union { + struct cdma_rw_wr rw; + struct cdma_cas_wr cas; + struct cdma_faa_wr faa; + }; + struct cdma_jfs_wr *next; +}; + +struct cdma_jfs { + struct cdma_base_jfs base_jfs; + struct cdma_dev *dev; + struct cdma_jetty_queue sq; + struct cdma_jfs_cfg cfg; + u64 jfs_addr; + u32 id; + u32 queue_id; + bool is_kernel; + refcount_t ae_ref_cnt; + struct completion ae_comp; +}; + +struct cdma_jfs_ctx { + /* DW0 */ + u32 ta_timeout : 2; + u32 rnr_retry_num : 3; + u32 type : 3; + u32 sqe_bb_shift : 4; + u32 sl : 4; + u32 state : 3; + u32 jfs_mode : 1; + u32 sqe_token_id_l : 12; + /* DW1 */ + u32 sqe_token_id_h : 8; + u32 err_mode : 1; + u32 rsv0 : 1; + u32 cmp_odr : 1; + u32 rsv1 : 1; + u32 sqe_base_addr_l : 20; + /* DW2 */ + u32 sqe_base_addr_h; + /* DW3 */ + u32 rsv2; + /* DW4 */ + u32 tx_jfcn : 20; + u32 jfrn_l : 12; + /* DW5 */ + u32 jfrn_h : 8; + u32 rsv3 : 4; + u32 rx_jfcn : 20; + /* DW6 */ + u32 seid_idx : 10; + u32 rsv4 : 22; + /* DW7 */ + u32 user_data_l; + /* DW8 */ + u32 user_data_h; + /* DW9 */ + u32 sqe_pos : 1; + u32 sqe_pld_pos : 1; + u32 sqe_pld_tokenid : 20; + u32 rsv5 : 10; + /* DW10 */ + u32 tpn : 24; + u32 rsv6 : 8; + /* DW11 */ + u32 rmt_eid : 20; + u32 rsv7 : 12; + /* DW12 */ + u32 rmt_tokenid : 20; + u64 rsv9 : 12; + /* DW13-DW15 */ + u32 rsv12[3]; + /* DW16 */ + u32 next_send_ssn : 16; + u32 src_order_wqe : 16; + /* DW17 */ + u32 src_order_ssn : 16; + u32 src_order_sgme_cnt : 16; + /* DW18 */ + u32 src_order_sgme_send_cnt : 16; + u32 ci : 16; + /* DW19 */ + u32 rsv13; + /* DW20 */ + u32 pi : 16; + u32 sq_db_doing : 1; + u32 ost_rce_credit : 15; + /* DW21 */ + u32 sq_db_retrying : 1; + u32 wmtp_rsv0 : 31; + /* DW22 */ + u32 wait_ack_timeout : 1; + u32 wait_rnr_timeout : 1; + u32 cqe_ie : 1; + u32 cqe_sz : 1; + u32 wmtp_rsv1 : 28; + /* DW23 */ + u32 wml_rsv1; + /* DW24 */ + u32 next_rcv_ssn : 16; + u32 next_cpl_bb_idx : 16; + /* DW25 */ + u32 next_cpl_sgmt_num : 20; + u32 we_rsv0 : 12; + /* DW26 */ + u32 next_cpl_bb_num : 4; + u32 next_cpl_cqe_en : 1; + u32 next_cpl_info_vld : 1; + u32 rpting_cqe : 1; + u32 not_rpt_cqe : 1; + u32 flush_ssn : 16; + u32 flush_ssn_vld : 1; + u32 flush_vld : 1; + u32 flush_cqe_done : 1; + u32 we_rsv1 : 5; + /* DW27 */ + u32 rcved_cont_ssn_num : 20; + u32 we_rsv2 : 12; + /* DW28 */ + u32 sq_timer; + /* DW29 */ + u32 rnr_cnt : 3; + u32 abt_ssn : 16; + u32 abt_ssn_vld : 1; + u32 taack_timeout_flag : 1; + u32 we_rsv3 : 9; + u32 err_type_l : 2; + /* DW30 */ + u32 err_type_h : 7; + u32 sq_flush_ssn : 16; + u32 we_rsv4 : 9; + /* DW31 */ + u32 avail_sgmt_ost : 10; + u32 read_op_cnt : 10; + u32 we_rsv5 : 12; + /* DW32 - DW63 */ + u32 taack_nack_bm[32]; +}; + +static inline struct cdma_jfs *to_cdma_jfs(struct cdma_base_jfs *jfs) +{ + return container_of(jfs, struct cdma_jfs, base_jfs); +} + +struct cdma_base_jfs *cdma_create_jfs(struct cdma_dev *cdev, + struct cdma_jfs_cfg *cfg, + struct cdma_udata *udata); +int cdma_delete_jfs(struct cdma_dev *cdev, u32 jfs_id); +int cdma_post_jfs_wr(struct cdma_jfs *jfs, struct cdma_jfs_wr *wr, + struct cdma_jfs_wr **bad_wr); + +#endif diff --git a/drivers/ub/cdma/cdma_main.c b/drivers/ub/cdma/cdma_main.c new file mode 100644 index 0000000000000000000000000000000000000000..8ec5849ade39514c1241cd6959583851ba04f9e7 --- /dev/null +++ b/drivers/ub/cdma/cdma_main.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define pr_fmt(fmt) "CDMA: " fmt +#define dev_fmt pr_fmt + +#include + +#include "cdma.h" +#include "cdma_dev.h" +#include "cdma_chardev.h" +#include +#include "cdma_eq.h" +#include "cdma_debugfs.h" +#include "cdma_cmd.h" +#include "cdma_types.h" +#include "cdma_mmap.h" +#include "cdma_context.h" +#include "cdma_uobj.h" +#include "cdma_event.h" + +static bool is_rmmod; +DEFINE_MUTEX(g_cdma_reset_mutex); + +/* Enabling jfc_arm_mode will cause jfc to report cqe; otherwise, it will not. */ +uint jfc_arm_mode; +module_param(jfc_arm_mode, uint, 0444); +MODULE_PARM_DESC(jfc_arm_mode, + "Set the ARM mode of the JFC, default: 0(0:Always ARM, others: NO ARM)"); + +bool cqe_mode = true; +module_param(cqe_mode, bool, 0444); +MODULE_PARM_DESC(cqe_mode, "Set cqe reporting mode, default: 1 (0:BY_COUNT, 1:BY_CI_PI_GAP)"); + +struct class *cdma_cdev_class; + +static int cdma_register_event(struct auxiliary_device *adev) +{ + int ret; + + ret = cdma_reg_ae_event(adev); + if (ret) + return ret; + + ret = cdma_reg_ce_event(adev); + if (ret) + goto err_ce_register; + + return 0; + +err_ce_register: + cdma_unreg_ae_event(adev); + + return ret; +} + +static inline void cdma_unregister_event(struct auxiliary_device *adev) +{ + cdma_unreg_ce_event(adev); + cdma_unreg_ae_event(adev); +} + +static void cdma_reset_unmap_vma_pages(struct cdma_dev *cdev, bool is_reset) +{ + struct cdma_file *cfile; + + mutex_lock(&cdev->file_mutex); + list_for_each_entry(cfile, &cdev->file_list, list) { + mutex_lock(&cfile->ctx_mutex); + cdma_unmap_vma_pages(cfile); + if (is_reset && cfile->uctx != NULL) + cfile->uctx->invalid = true; + mutex_unlock(&cfile->ctx_mutex); + } + mutex_unlock(&cdev->file_mutex); +} + +static void cdma_client_handler(struct cdma_dev *cdev, + enum cdma_client_ops client_ops) +{ + struct dma_client *client; + + down_write(&g_clients_rwsem); + list_for_each_entry(client, &g_client_list, list_node) { + switch (client_ops) { + case CDMA_CLIENT_STOP: + if (client->stop) + client->stop(cdev->eid); + break; + case CDMA_CLIENT_REMOVE: + if (client->remove) + client->remove(cdev->eid); + break; + case CDMA_CLIENT_ADD: + if (client->add && client->add(cdev->eid)) + dev_warn(&cdev->adev->dev, "add eid:0x%x, cdev for client:%s failed.\n", + cdev->eid, client->client_name); + break; + } + } + up_write(&g_clients_rwsem); +} + +static void cdma_reset_down(struct auxiliary_device *adev) +{ + struct cdma_dev *cdev; + + mutex_lock(&g_cdma_reset_mutex); + cdev = get_cdma_dev(adev); + if (!cdev || cdev->status == CDMA_SUSPEND) { + dev_warn(&adev->dev, "cdma device is not ready.\n"); + mutex_unlock(&g_cdma_reset_mutex); + return; + } + + cdev->status = CDMA_SUSPEND; + cdma_cmd_flush(cdev); + cdma_reset_unmap_vma_pages(cdev, true); + cdma_client_handler(cdev, CDMA_CLIENT_STOP); + cdma_unregister_event(adev); + cdma_dbg_uninit(adev); + mutex_unlock(&g_cdma_reset_mutex); +} + +static void cdma_reset_uninit(struct auxiliary_device *adev) +{ + enum ubase_reset_stage stage; + struct cdma_dev *cdev; + + mutex_lock(&g_cdma_reset_mutex); + cdev = get_cdma_dev(adev); + if (!cdev) { + dev_info(&adev->dev, "cdma device is not exist.\n"); + mutex_unlock(&g_cdma_reset_mutex); + return; + } + + stage = ubase_get_reset_stage(adev); + if (stage == UBASE_RESET_STAGE_UNINIT && cdev->status == CDMA_SUSPEND) { + cdma_client_handler(cdev, CDMA_CLIENT_REMOVE); + cdma_destroy_dev(cdev, is_rmmod); + } + mutex_unlock(&g_cdma_reset_mutex); +} + +static int cdma_init_dev_info(struct auxiliary_device *auxdev, struct cdma_dev *cdev) +{ + int ret; + + ret = cdma_register_event(auxdev); + if (ret) + return ret; + + /* query eu failure does not affect driver loading, as eu can be updated. */ + ret = cdma_ctrlq_query_eu(cdev); + if (ret) + dev_warn(&auxdev->dev, "query eu failed, ret = %d.\n", ret); + + ret = cdma_dbg_init(auxdev); + if (ret) + dev_warn(&auxdev->dev, "init cdma debugfs failed, ret = %d.\n", + ret); + + return 0; +} + +static void cdma_free_cfile_uobj(struct cdma_dev *cdev) +{ + struct cdma_file *cfile, *next_cfile; + struct cdma_jfae *jfae; + + mutex_lock(&cdev->file_mutex); + list_for_each_entry_safe(cfile, next_cfile, &cdev->file_list, list) { + list_del(&cfile->list); + mutex_lock(&cfile->ctx_mutex); + cdma_cleanup_context_uobj(cfile, CDMA_REMOVE_DRIVER_REMOVE); + cfile->cdev = NULL; + if (cfile->uctx) { + jfae = (struct cdma_jfae *)cfile->uctx->jfae; + if (jfae) + wake_up_interruptible(&jfae->jfe.poll_wait); + cdma_cleanup_context_res(cfile->uctx); + } + cfile->uctx = NULL; + mutex_unlock(&cfile->ctx_mutex); + } + mutex_unlock(&cdev->file_mutex); +} + +static int cdma_init_dev(struct auxiliary_device *auxdev) +{ + struct cdma_dev *cdev; + bool is_remove = true; + int ret; + + dev_dbg(&auxdev->dev, "%s called, matched aux dev(%s.%u).\n", + __func__, auxdev->name, auxdev->id); + + cdev = cdma_create_dev(auxdev); + if (!cdev) + return -ENOMEM; + + ret = cdma_create_chardev(cdev); + if (ret) { + cdma_destroy_dev(cdev, is_remove); + return ret; + } + + ret = cdma_init_dev_info(auxdev, cdev); + if (ret) { + cdma_destroy_chardev(cdev); + cdma_destroy_dev(cdev, is_remove); + return ret; + } + + cdma_client_handler(cdev, CDMA_CLIENT_ADD); + return ret; +} + +static void cdma_uninit_dev(struct auxiliary_device *auxdev) +{ + struct cdma_dev *cdev; + int ret; + + dev_dbg(&auxdev->dev, "%s called, matched aux dev(%s.%u).\n", + __func__, auxdev->name, auxdev->id); + + mutex_lock(&g_cdma_reset_mutex); + cdev = dev_get_drvdata(&auxdev->dev); + if (!cdev) { + dev_err(&auxdev->dev, "get drvdata from ubase failed.\n"); + ubase_reset_unregister(auxdev); + mutex_unlock(&g_cdma_reset_mutex); + return; + } + + cdev->status = CDMA_SUSPEND; + cdma_cmd_flush(cdev); + cdma_client_handler(cdev, CDMA_CLIENT_STOP); + cdma_client_handler(cdev, CDMA_CLIENT_REMOVE); + cdma_reset_unmap_vma_pages(cdev, false); + + if (!is_rmmod) { + ret = ubase_deactivate_dev(auxdev); + dev_info(&auxdev->dev, "ubase deactivate dev ret = %d.\n", ret); + } + + ubase_reset_unregister(auxdev); + cdma_dbg_uninit(auxdev); + cdma_unregister_event(auxdev); + cdma_destroy_chardev(cdev); + cdma_free_cfile_uobj(cdev); + cdma_destroy_dev(cdev, true); + mutex_unlock(&g_cdma_reset_mutex); +} + +static void cdma_reset_init(struct auxiliary_device *adev) +{ + struct cdma_dev *cdev; + + mutex_lock(&g_cdma_reset_mutex); + cdev = get_cdma_dev(adev); + if (!cdev) { + dev_err(&adev->dev, "cdma device is not exist.\n"); + mutex_unlock(&g_cdma_reset_mutex); + return; + } + + if (cdma_register_crq_event(adev)) { + mutex_unlock(&g_cdma_reset_mutex); + return; + } + + if (cdma_create_arm_db_page(cdev)) + goto unregister_crq; + + if (cdma_init_dev_info(adev, cdev)) + goto destory_arm_db_page; + + idr_init(&cdev->ctx_idr); + spin_lock_init(&cdev->ctx_lock); + atomic_set(&cdev->cmdcnt, 1); + cdev->status = CDMA_NORMAL; + cdma_client_handler(cdev, CDMA_CLIENT_ADD); + mutex_unlock(&g_cdma_reset_mutex); + return; + +destory_arm_db_page: + cdma_destroy_arm_db_page(cdev); +unregister_crq: + cdma_unregister_crq_event(adev); + mutex_unlock(&g_cdma_reset_mutex); +} + +static void cdma_reset_handler(struct auxiliary_device *adev, + enum ubase_reset_stage stage) +{ + if (!adev) + return; + + switch (stage) { + case UBASE_RESET_STAGE_DOWN: + cdma_reset_down(adev); + break; + case UBASE_RESET_STAGE_UNINIT: + cdma_reset_uninit(adev); + break; + case UBASE_RESET_STAGE_INIT: + if (!is_rmmod) + cdma_reset_init(adev); + break; + default: + break; + } +} + +static int cdma_probe(struct auxiliary_device *auxdev, + const struct auxiliary_device_id *auxdev_id) +{ + int ret; + + ret = cdma_init_dev(auxdev); + if (ret) + return ret; + + ubase_reset_register(auxdev, cdma_reset_handler); + + return 0; +} + +static void cdma_remove(struct auxiliary_device *auxdev) +{ + cdma_uninit_dev(auxdev); + pr_info("cdma device remove success.\n"); +} + +static const struct auxiliary_device_id cdma_id_table[] = { + { + .name = UBASE_ADEV_NAME ".cdma", + }, + {} +}; +MODULE_DEVICE_TABLE(auxiliary, cdma_id_table); + +static struct auxiliary_driver cdma_driver = { + .probe = cdma_probe, + .remove = cdma_remove, + .name = "cdma", + .id_table = cdma_id_table, +}; + +static int __init cdma_init(void) +{ + int ret; + + cdma_cdev_class = class_create("cdma"); + if (IS_ERR(cdma_cdev_class)) { + pr_err("create cdma class failed.\n"); + return PTR_ERR(cdma_cdev_class); + } + + ret = auxiliary_driver_register(&cdma_driver); + if (ret) { + pr_err("auxiliary register failed.\n"); + goto free_class; + } + + return 0; + +free_class: + class_destroy(cdma_cdev_class); + + return ret; +} + +static void __exit cdma_exit(void) +{ + is_rmmod = true; + auxiliary_driver_unregister(&cdma_driver); + class_destroy(cdma_cdev_class); +} + +module_init(cdma_init); +module_exit(cdma_exit); +MODULE_LICENSE("GPL"); +MODULE_VERSION("1.0"); +MODULE_DESCRIPTION("Hisilicon UBus Crystal DMA Driver"); diff --git a/drivers/ub/cdma/cdma_mbox.c b/drivers/ub/cdma/cdma_mbox.c new file mode 100644 index 0000000000000000000000000000000000000000..194eba8a920d81d2dccc845938d3e2d86db4258d --- /dev/null +++ b/drivers/ub/cdma/cdma_mbox.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "CDMA: " fmt + +#include "cdma_mbox.h" + +static int cdma_post_mailbox(struct cdma_dev *cdev, struct ubase_mbx_attr *attr, + struct ubase_cmd_mailbox *mailbox) +{ + int ret; + + ret = ubase_hw_upgrade_ctx_ex(cdev->adev, attr, mailbox); + if (ret) + dev_err(cdev->dev, + "send mailbox err, tag = 0x%x, op = %u, mbx_ue_id = %u.\n", + attr->tag, attr->op, attr->mbx_ue_id); + + return ret; +} + +int cdma_post_mailbox_ctx(struct cdma_dev *cdev, void *ctx, u32 size, + struct ubase_mbx_attr *attr) +{ + struct ubase_cmd_mailbox *mailbox; + int ret; + + mailbox = cdma_alloc_cmd_mailbox(cdev); + if (!mailbox) { + dev_err(cdev->dev, "alloc mailbox failed, opcode = %u.\n", + attr->op); + return -ENOMEM; + } + + if (ctx && size) + memcpy(mailbox->buf, ctx, size); + + ret = cdma_post_mailbox(cdev, attr, mailbox); + + cdma_free_cmd_mailbox(cdev, mailbox); + + return ret; +} + +struct ubase_cmd_mailbox *cdma_mailbox_query_ctx(struct cdma_dev *cdev, + struct ubase_mbx_attr *attr) +{ + struct ubase_cmd_mailbox *mailbox; + int ret; + + mailbox = cdma_alloc_cmd_mailbox(cdev); + if (!mailbox) { + dev_err(cdev->dev, "alloc mailbox failed, opcode = %u.\n", + attr->op); + return NULL; + } + + ret = cdma_post_mailbox(cdev, attr, mailbox); + if (ret) { + cdma_free_cmd_mailbox(cdev, mailbox); + return NULL; + } + + return mailbox; +} diff --git a/drivers/ub/cdma/cdma_mbox.h b/drivers/ub/cdma/cdma_mbox.h new file mode 100644 index 0000000000000000000000000000000000000000..e8a00f5c9b976ea9395e35a28f758e12f11f1203 --- /dev/null +++ b/drivers/ub/cdma/cdma_mbox.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_MBOX_H__ +#define __CDMA_MBOX_H__ + +#include "cdma.h" +#include + +enum { + /* JFS CMDS */ + CDMA_CMD_CREATE_JFS_CONTEXT = 0x04, + CDMA_CMD_MODIFY_JFS_CONTEXT = 0x05, + CDMA_CMD_QUERY_JFS_CONTEXT = 0x06, + CDMA_CMD_DESTROY_JFS_CONTEXT = 0x07, + + /* JFC CMDS */ + CDMA_CMD_CREATE_JFC_CONTEXT = 0x24, + CDMA_CMD_QUERY_JFC_CONTEXT = 0x26, + CDMA_CMD_DESTROY_JFC_CONTEXT = 0x27, +}; + +/* The mailbox operation is as follows: */ +static inline void cdma_fill_mbx_attr(struct ubase_mbx_attr *attr, u32 tag, + u8 op, u8 mbx_ue_id) +{ + ubase_fill_mbx_attr(attr, tag, op, mbx_ue_id); +} + +static inline struct ubase_cmd_mailbox *cdma_alloc_cmd_mailbox(struct cdma_dev *cdev) +{ + return ubase_alloc_cmd_mailbox(cdev->adev); +} + +static inline void cdma_free_cmd_mailbox(struct cdma_dev *cdev, + struct ubase_cmd_mailbox *mailbox) +{ + ubase_free_cmd_mailbox(cdev->adev, mailbox); +} + +int cdma_post_mailbox_ctx(struct cdma_dev *cdev, void *ctx, u32 size, + struct ubase_mbx_attr *attr); +struct ubase_cmd_mailbox *cdma_mailbox_query_ctx(struct cdma_dev *cdev, + struct ubase_mbx_attr *attr); + +#endif /* CDMA_MBOX_H */ diff --git a/drivers/ub/cdma/cdma_mmap.c b/drivers/ub/cdma/cdma_mmap.c new file mode 100644 index 0000000000000000000000000000000000000000..eaef6a9a4152d72bc3e9598728873211930bee1a --- /dev/null +++ b/drivers/ub/cdma/cdma_mmap.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define pr_fmt(fmt) "CDMA: " fmt + +#include +#include +#include "cdma_mmap.h" + +void cdma_umap_priv_init(struct cdma_umap_priv *priv, + struct vm_area_struct *vma) +{ + struct cdma_file *cfile = (struct cdma_file *)vma->vm_file->private_data; + + priv->vma = vma; + vma->vm_private_data = priv; + + mutex_lock(&cfile->umap_mutex); + list_add(&priv->node, &cfile->umaps_list); + mutex_unlock(&cfile->umap_mutex); +} + +/* thanks to drivers/infiniband/core/ib_core_uverbs.c */ +void cdma_unmap_vma_pages(struct cdma_file *cfile) +{ + struct cdma_umap_priv *priv, *next_priv; + struct mm_struct *mm = NULL; + struct vm_area_struct *vma; + int ret; + + while (1) { + mm = NULL; + mutex_lock(&cfile->umap_mutex); + list_for_each_entry_safe(priv, next_priv, &cfile->umaps_list, node) { + mm = priv->vma->vm_mm; + ret = mmget_not_zero(mm); + if (!ret) { + list_del_init(&priv->node); + mm = NULL; + continue; + } + break; + } + mutex_unlock(&cfile->umap_mutex); + if (!mm) + return; + + mutex_lock(&cfile->umap_mutex); + list_for_each_entry_safe(priv, next_priv, &cfile->umaps_list, node) { + vma = priv->vma; + if (vma->vm_mm != mm) + continue; + list_del_init(&priv->node); + zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start); + } + mutex_unlock(&cfile->umap_mutex); + + mmput(mm); + } +} + +static void cdma_umap_open(struct vm_area_struct *vma) +{ + struct cdma_umap_priv *priv; + + priv = kzalloc(sizeof(struct cdma_umap_priv), GFP_KERNEL); + if (!priv) + goto out_zap; + + cdma_umap_priv_init(priv, vma); + + return; + +out_zap: + vma->vm_private_data = NULL; + zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start); +} + +static void cdma_umap_close(struct vm_area_struct *vma) +{ + struct cdma_umap_priv *priv = (struct cdma_umap_priv *)vma->vm_private_data; + struct cdma_file *cfile = (struct cdma_file *)vma->vm_file->private_data; + + if (!priv) + return; + + mutex_lock(&cfile->umap_mutex); + list_del(&priv->node); + mutex_unlock(&cfile->umap_mutex); + kfree(priv); + vma->vm_private_data = NULL; + + pr_info("cdma umap close success.\n"); +} + +static vm_fault_t cdma_umap_fault(struct vm_fault *vmf) +{ + struct cdma_umap_priv *priv = (struct cdma_umap_priv *)vmf->vma->vm_private_data; + struct cdma_file *cfile = (struct cdma_file *)vmf->vma->vm_file->private_data; + vm_fault_t ret = 0; + + if (!priv) + return VM_FAULT_SIGBUS; + + if (!(vmf->vma->vm_flags & (VM_WRITE | VM_MAYWRITE))) { + vmf->page = ZERO_PAGE(0); + get_page(vmf->page); + return 0; + } + + mutex_lock(&cfile->umap_mutex); + if (!cfile->fault_page) + cfile->fault_page = alloc_pages(vmf->gfp_mask | __GFP_ZERO, 0); + + if (cfile->fault_page) { + vmf->page = cfile->fault_page; + get_page(vmf->page); + } else { + ret = VM_FAULT_SIGBUS; + } + mutex_unlock(&cfile->umap_mutex); + + return ret; +} + +static int cdma_umap_remap(struct vm_area_struct *vma) +{ + pr_err("cdma umap remap is not permitted.\n"); + return -EINVAL; +} + +static int cdma_umap_can_split(struct vm_area_struct *vma, unsigned long addr) +{ + pr_err("cdma umap split is not permitted.\n"); + return -EINVAL; +} + +static const struct vm_operations_struct g_cdma_umap_ops = { + .open = cdma_umap_open, + .close = cdma_umap_close, + .fault = cdma_umap_fault, + .mremap = cdma_umap_remap, + .may_split = cdma_umap_can_split, +}; + +const struct vm_operations_struct *cdma_get_umap_ops(void) +{ + return (const struct vm_operations_struct *)&g_cdma_umap_ops; +} diff --git a/drivers/ub/cdma/cdma_mmap.h b/drivers/ub/cdma/cdma_mmap.h new file mode 100644 index 0000000000000000000000000000000000000000..0dd6c609a85ee0ac6b6015366358a6325ba5af8a --- /dev/null +++ b/drivers/ub/cdma/cdma_mmap.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_MMAP_H__ +#define __CDMA_MMAP_H__ + +#include +#include "cdma_types.h" + +void cdma_unmap_vma_pages(struct cdma_file *cfile); +const struct vm_operations_struct *cdma_get_umap_ops(void); +void cdma_umap_priv_init(struct cdma_umap_priv *priv, struct vm_area_struct *vma); + +#endif /* CDMA_MMAP_H */ diff --git a/drivers/ub/cdma/cdma_queue.c b/drivers/ub/cdma/cdma_queue.c new file mode 100644 index 0000000000000000000000000000000000000000..9b03baef162ceb142cdac118517afa6a2a68682c --- /dev/null +++ b/drivers/ub/cdma/cdma_queue.c @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "CDMA: " fmt + +#include "cdma_common.h" +#include "cdma_context.h" +#include "cdma_jfc.h" +#include "cdma_jfs.h" +#include "cdma_tp.h" +#include "cdma_queue.h" +#include "cdma.h" + +struct cdma_queue *cdma_find_queue(struct cdma_dev *cdev, u32 queue_id) +{ + struct cdma_queue *queue; + + spin_lock(&cdev->queue_table.lock); + queue = (struct cdma_queue *)idr_find(&cdev->queue_table.idr_tbl.idr, + queue_id); + spin_unlock(&cdev->queue_table.lock); + + return queue; +} + +static void cdma_k_assemble_jfs_cfg(struct cdma_jfs_cfg *jfs_cfg, + struct cdma_dev *cdev, + u32 eid_index, + struct queue_cfg *cfg, + struct cdma_queue *queue) +{ + jfs_cfg->eid_index = eid_index; + jfs_cfg->max_rsge = cdev->base.attr.dev_cap.max_jfs_rsge; + jfs_cfg->max_sge = cdev->base.attr.dev_cap.max_jfs_sge; + jfs_cfg->depth = cfg->queue_depth; + jfs_cfg->err_timeout = CDMA_TYPICAL_ERR_TIMEOUT; + jfs_cfg->priority = cfg->priority; + jfs_cfg->rnr_retry = CDMA_TYPICAL_RNR_RETRY; + jfs_cfg->rmt_eid = cfg->rmt_eid.dw0; + jfs_cfg->queue_id = queue->id; + jfs_cfg->trans_mode = cfg->trans_mode; +} + +static void cdma_k_assemble_jfc_cfg(struct cdma_jfc_cfg *jfc_cfg, + struct queue_cfg *cfg, + struct cdma_queue *queue) +{ + jfc_cfg->depth = cfg->queue_depth; + jfc_cfg->queue_id = queue->id; +} + +static void cdma_k_assemble_tp_cfg(struct cdma_tp_cfg *tp_cfg, + struct cdma_dev *cdev, + struct queue_cfg *cfg) +{ + tp_cfg->seid = cdev->base.attr.eu.eid.dw0; + tp_cfg->dcna = cfg->dcna; + tp_cfg->deid = cfg->rmt_eid.dw0; +} + +static int cdma_create_queue_res(struct cdma_dev *cdev, struct queue_cfg *cfg, + struct cdma_queue *queue, u32 eid_index) +{ + struct cdma_jfc_cfg jfc_cfg = { 0 }; + struct cdma_jfs_cfg jfs_cfg = { 0 }; + struct cdma_tp_cfg tp_cfg = { 0 }; + int ret; + + cdma_k_assemble_jfs_cfg(&jfs_cfg, cdev, eid_index, cfg, queue); + cdma_k_assemble_jfc_cfg(&jfc_cfg, cfg, queue); + cdma_k_assemble_tp_cfg(&tp_cfg, cdev, cfg); + + queue->jfc = cdma_create_jfc(cdev, &jfc_cfg, NULL); + if (!queue->jfc) { + dev_err(cdev->dev, "create jfc failed.\n"); + return -EFAULT; + } + + queue->tp = cdma_create_ctp(cdev, &tp_cfg); + if (!queue->tp) { + dev_err(cdev->dev, "create tp failed.\n"); + ret = -EFAULT; + goto delete_jfc; + } + + jfs_cfg.tpn = queue->tp->tpn; + jfs_cfg.jfc_id = queue->jfc->id; + queue->jfs = cdma_create_jfs(cdev, &jfs_cfg, NULL); + if (!queue->jfs) { + dev_err(cdev->dev, "create jfs failed.\n"); + ret = -EFAULT; + goto delete_tp; + } + + queue->jfs_id = queue->jfs->id; + queue->jfc_id = queue->jfc->id; + + dev_dbg(cdev->dev, "set queue %u jfs id: %u, jfc id: %u.\n", + queue->id, queue->jfs_id, queue->jfc_id); + + return 0; + +delete_tp: + cdma_delete_ctp(cdev, queue->tp->tp_id); +delete_jfc: + cdma_delete_jfc(cdev, queue->jfc->id, NULL); + + return ret; +} + +static void cdma_delete_queue_res(struct cdma_dev *cdev, + struct cdma_queue *queue) +{ + cdma_delete_jfs(cdev, queue->jfs->id); + queue->jfs = NULL; + cdma_delete_ctp(cdev, queue->tp->tp_id); + queue->tp = NULL; + cdma_delete_jfc(cdev, queue->jfc->id, NULL); + queue->jfc = NULL; +} + +static int cdma_alloc_queue_id(struct cdma_dev *cdev, struct cdma_queue *queue) +{ + struct cdma_table *queue_tbl = &cdev->queue_table; + int id; + + idr_preload(GFP_KERNEL); + spin_lock(&queue_tbl->lock); + id = idr_alloc(&queue_tbl->idr_tbl.idr, queue, queue_tbl->idr_tbl.min, + queue_tbl->idr_tbl.max, GFP_NOWAIT); + if (id < 0) + dev_err(cdev->dev, "alloc queue id failed.\n"); + spin_unlock(&queue_tbl->lock); + idr_preload_end(); + + return id; +} + +static void cdma_delete_queue_id(struct cdma_dev *cdev, int queue_id) +{ + struct cdma_table *queue_tbl = &cdev->queue_table; + + spin_lock(&queue_tbl->lock); + idr_remove(&queue_tbl->idr_tbl.idr, queue_id); + spin_unlock(&queue_tbl->lock); +} + +struct cdma_queue *cdma_create_queue(struct cdma_dev *cdev, + struct cdma_context *uctx, + struct queue_cfg *cfg, u32 eid_index, + bool is_kernel) +{ + struct cdma_queue *queue; + int ret; + int id; + + queue = kzalloc(sizeof(*queue), GFP_KERNEL); + if (!queue) + return NULL; + + id = cdma_alloc_queue_id(cdev, queue); + if (id < 0) { + kfree(queue); + return NULL; + } + + queue->ctx = uctx; + queue->id = id; + queue->cfg = *cfg; + + if (is_kernel) { + ret = cdma_create_queue_res(cdev, cfg, queue, eid_index); + if (ret) { + dev_err(cdev->dev, "create queue res failed.\n"); + cdma_delete_queue_id(cdev, id); + kfree(queue); + return NULL; + } + queue->is_kernel = true; + } + + return queue; +} + +int cdma_delete_queue(struct cdma_dev *cdev, u32 queue_id) +{ + struct cdma_queue *queue; + + if (!cdev) + return -EINVAL; + + if (queue_id >= cdev->caps.queue.start_idx + cdev->caps.queue.max_cnt) { + dev_err(cdev->dev, + "queue id invalid, queue_id = %u, start_idx = %u, max_cnt = %u.\n", + queue_id, cdev->caps.queue.start_idx, + cdev->caps.queue.max_cnt); + return -EINVAL; + } + + queue = cdma_find_queue(cdev, queue_id); + if (!queue) { + dev_err(cdev->dev, "get queue from table failed, id = %u.\n", + queue_id); + return -EINVAL; + } + + cdma_delete_queue_id(cdev, queue_id); + + if (queue->is_kernel) + cdma_delete_queue_res(cdev, queue); + kfree(queue); + + return 0; +} + +void cdma_set_queue_res(struct cdma_dev *cdev, struct cdma_queue *queue, + enum cdma_queue_res_type type, void *res) +{ + dev_dbg(cdev->dev, + "set queue %u resource type = %u, null flag = %u.\n", + queue->id, type, res == NULL); + + spin_lock(&cdev->queue_table.lock); + switch (type) { + case QUEUE_RES_TP: + queue->tp = res; + break; + case QUEUE_RES_JFS: + queue->jfs = res; + if (queue->jfs) + queue->jfs_id = queue->jfs->id; + break; + case QUEUE_RES_JFC: + queue->jfc = res; + if (queue->jfc) + queue->jfc_id = queue->jfc->id; + break; + default: + break; + } + spin_unlock(&cdev->queue_table.lock); +} diff --git a/drivers/ub/cdma/cdma_queue.h b/drivers/ub/cdma/cdma_queue.h new file mode 100644 index 0000000000000000000000000000000000000000..08b24cb0b3fc437cbf3800e4a2827086cfb9d850 --- /dev/null +++ b/drivers/ub/cdma/cdma_queue.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_QUEUE_H__ +#define __CDMA_QUEUE_H__ + +#include + +struct cdma_dev; +struct cdma_context; + +enum cdma_queue_res_type { + QUEUE_RES_TP, + QUEUE_RES_JFS, + QUEUE_RES_JFC +}; + +struct cdma_queue { + struct cdma_base_jfc *jfc; + struct cdma_base_jfs *jfs; + struct cdma_base_tp *tp; + struct cdma_context *ctx; + u32 id; + struct queue_cfg cfg; + bool is_kernel; + struct list_head list; + u32 jfs_id; + u32 jfc_id; +}; + +struct cdma_queue *cdma_find_queue(struct cdma_dev *cdev, u32 queue_id); +struct cdma_queue *cdma_create_queue(struct cdma_dev *cdev, + struct cdma_context *uctx, + struct queue_cfg *cfg, u32 eid_index, + bool is_kernel); +int cdma_delete_queue(struct cdma_dev *cdev, u32 queue_id); +void cdma_set_queue_res(struct cdma_dev *cdev, struct cdma_queue *queue, + enum cdma_queue_res_type type, void *res); +#endif diff --git a/drivers/ub/cdma/cdma_segment.c b/drivers/ub/cdma/cdma_segment.c new file mode 100644 index 0000000000000000000000000000000000000000..6882d27cd70a86eef954d61bde798cf05f26219a --- /dev/null +++ b/drivers/ub/cdma/cdma_segment.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "CDMA: " fmt + +#include +#include "cdma_segment.h" +#include "cdma_context.h" + +static int cdma_alloc_seg_handle(struct cdma_dev *cdev, + struct cdma_segment *seg) +{ + struct cdma_table *seg_table = &cdev->seg_table; + int handle; + + idr_preload(GFP_KERNEL); + spin_lock(&seg_table->lock); + + handle = idr_alloc(&seg_table->idr_tbl.idr, seg, seg_table->idr_tbl.min, + seg_table->idr_tbl.max, GFP_NOWAIT); + if (handle < 0) + dev_err(cdev->dev, "alloc seg handle failed.\n"); + + spin_unlock(&seg_table->lock); + idr_preload_end(); + + return handle; +} + +static inline void cdma_free_seg_handle(struct cdma_dev *cdev, u64 handle) +{ + spin_lock(&cdev->seg_table.lock); + idr_remove(&cdev->seg_table.idr_tbl.idr, handle); + spin_unlock(&cdev->seg_table.lock); +} + +struct cdma_segment *cdma_register_seg(struct cdma_dev *cdev, + struct dma_seg_cfg *cfg, bool is_kernel) +{ + struct cdma_segment *seg; + int handle; + + seg = kzalloc(sizeof(*seg), GFP_KERNEL); + if (!seg) + return NULL; + + seg->umem = cdma_umem_get(cdev, cfg->sva, cfg->len, is_kernel); + if (IS_ERR_OR_NULL(seg->umem)) { + dev_err(cdev->dev, "pin seg failed\n"); + goto free_seg; + } + + handle = cdma_alloc_seg_handle(cdev, seg); + if (handle < 0) + goto unpin_umem; + + seg->base.handle = (u64)handle; + seg->base.token_value = cfg->token_value; + seg->base.sva = cfg->sva; + seg->base.len = cfg->len; + seg->base.token_value_valid = cfg->token_value_valid; + seg->is_kernel = is_kernel; + + return seg; + +unpin_umem: + cdma_umem_release(seg->umem, is_kernel); +free_seg: + kfree(seg); + + return NULL; +} + +void cdma_unregister_seg(struct cdma_dev *cdev, struct cdma_segment *seg) +{ + cdma_free_seg_handle(cdev, seg->base.handle); + cdma_umem_release(seg->umem, seg->is_kernel); + kfree(seg); +} + +int cdma_seg_grant(struct cdma_dev *cdev, struct cdma_segment *seg, + struct dma_seg_cfg *cfg) +{ + struct ummu_token_info token_info; + struct ummu_seg_attr seg_attr; + int ret; + + seg->base.tid = seg->ctx->tid; + seg->ksva = seg->ctx->sva; + + token_info.input = 0; + token_info.tokenVal = cfg->token_value; + seg_attr.token = &token_info; + seg_attr.e_bit = UMMU_EBIT_OFF; + + ret = ummu_sva_grant_range(seg->ksva, (void *)cfg->sva, cfg->len, + MAPT_PERM_RW, (void *)&seg_attr); + if (ret) + dev_err(cdev->dev, "grant seg failed, ret = %d.\n", ret); + + return ret; +} + +void cdma_seg_ungrant(struct cdma_segment *seg) +{ + struct ummu_token_info token_info = { 0 }; + + token_info.tokenVal = seg->base.token_value; + + ummu_sva_ungrant_range(seg->ksva, (void *)seg->base.sva, + seg->base.len, &token_info); +} + +struct dma_seg *cdma_import_seg(struct dma_seg_cfg *cfg) +{ + struct dma_seg *seg; + + seg = kzalloc(sizeof(*seg), GFP_KERNEL); + if (!seg) + return NULL; + + seg->sva = cfg->sva; + seg->len = cfg->len; + seg->token_value = cfg->token_value; + + return seg; +} + +void cdma_unimport_seg(struct dma_seg *seg) +{ + kfree(seg); +} diff --git a/drivers/ub/cdma/cdma_segment.h b/drivers/ub/cdma/cdma_segment.h new file mode 100644 index 0000000000000000000000000000000000000000..113e357fcedd8ab7112018b3de09d1e139b63178 --- /dev/null +++ b/drivers/ub/cdma/cdma_segment.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_SEGMENT_H__ +#define __CDMA_SEGMENT_H__ + +#include "cdma_common.h" +#include + +struct cdma_dev; + +struct cdma_segment { + struct dma_seg base; + struct iommu_sva *ksva; + struct cdma_umem *umem; + struct cdma_context *ctx; + bool is_kernel; + struct list_head list; +}; + +static inline struct cdma_segment *to_cdma_seg(struct dma_seg *seg) +{ + return container_of(seg, struct cdma_segment, base); +} + +struct cdma_segment *cdma_register_seg(struct cdma_dev *cdev, + struct dma_seg_cfg *cfg, bool is_kernel); +void cdma_unregister_seg(struct cdma_dev *cdev, struct cdma_segment *seg); +int cdma_seg_grant(struct cdma_dev *cdev, struct cdma_segment *seg, + struct dma_seg_cfg *cfg); +void cdma_seg_ungrant(struct cdma_segment *seg); +struct dma_seg *cdma_import_seg(struct dma_seg_cfg *cfg); +void cdma_unimport_seg(struct dma_seg *seg); + +#endif /* CDMA_SEGMENT_H */ diff --git a/drivers/ub/cdma/cdma_tid.c b/drivers/ub/cdma/cdma_tid.c new file mode 100644 index 0000000000000000000000000000000000000000..6e38b72bc8e0c3f9f5f6819175e806a568da3334 --- /dev/null +++ b/drivers/ub/cdma/cdma_tid.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "CDMA: " fmt + +#include +#include + +#include "cdma.h" +#include "cdma_tid.h" + +int cdma_alloc_dev_tid(struct cdma_dev *cdev) +{ + struct ummu_seg_attr seg_attr = { + .token = NULL, + .e_bit = UMMU_EBIT_ON, + }; + struct ummu_param drvdata = { + .mode = MAPT_MODE_TABLE, + }; + int ret; + + ret = iommu_dev_enable_feature(cdev->dev, IOMMU_DEV_FEAT_KSVA); + if (ret) { + dev_err(cdev->dev, "enable ksva failed, ret = %d.\n", ret); + return ret; + } + + ret = iommu_dev_enable_feature(cdev->dev, IOMMU_DEV_FEAT_SVA); + if (ret) { + dev_err(cdev->dev, "enable sva failed, ret = %d.\n", ret); + goto err_sva_enable_dev; + } + + cdev->ksva = ummu_ksva_bind_device(cdev->dev, &drvdata); + if (!cdev->ksva) { + dev_err(cdev->dev, "ksva bind device failed.\n"); + ret = -EINVAL; + goto err_ksva_bind_device; + } + + ret = ummu_get_tid(cdev->dev, cdev->ksva, &cdev->tid); + if (ret) { + dev_err(cdev->dev, "get tid for cdma device failed.\n"); + goto err_get_tid; + } + + ret = ummu_sva_grant_range(cdev->ksva, 0, CDMA_MAX_GRANT_SIZE, + UMMU_DEV_WRITE | UMMU_DEV_READ, + &seg_attr); + if (ret) { + dev_err(cdev->dev, "sva grant range for cdma device failed.\n"); + goto err_get_tid; + } + + return 0; + +err_get_tid: + ummu_ksva_unbind_device(cdev->ksva); +err_ksva_bind_device: + if (iommu_dev_disable_feature(cdev->dev, IOMMU_DEV_FEAT_SVA)) + dev_warn(cdev->dev, "disable sva failed, ret = %d.\n", ret); +err_sva_enable_dev: + if (iommu_dev_disable_feature(cdev->dev, IOMMU_DEV_FEAT_KSVA)) + dev_warn(cdev->dev, "disable ksva failed, ret = %d.\n", ret); + + return ret; +} + +void cdma_free_dev_tid(struct cdma_dev *cdev) +{ + int ret; + + ret = ummu_sva_ungrant_range(cdev->ksva, 0, CDMA_MAX_GRANT_SIZE, NULL); + if (ret) + dev_warn(cdev->dev, + "sva ungrant range for cdma device failed, ret = %d.\n", + ret); + + ummu_ksva_unbind_device(cdev->ksva); + + ret = iommu_dev_disable_feature(cdev->dev, IOMMU_DEV_FEAT_SVA); + if (ret) + dev_warn(cdev->dev, "disable sva failed, ret = %d.\n", ret); + + ret = iommu_dev_disable_feature(cdev->dev, IOMMU_DEV_FEAT_KSVA); + if (ret) + dev_warn(cdev->dev, "disable ksva failed, ret = %d.\n", ret); +} diff --git a/drivers/ub/cdma/cdma_tid.h b/drivers/ub/cdma/cdma_tid.h new file mode 100644 index 0000000000000000000000000000000000000000..8bbd8c0c979a2062304c0b93bd5581565ec19cf7 --- /dev/null +++ b/drivers/ub/cdma/cdma_tid.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_TID_H__ +#define __CDMA_TID_H__ + +#include + +#define CDMA_MAX_GRANT_SIZE GENMASK(47, 12) + +struct cdma_dev; + +int cdma_alloc_dev_tid(struct cdma_dev *cdev); +void cdma_free_dev_tid(struct cdma_dev *cdev); + +#endif diff --git a/drivers/ub/cdma/cdma_tp.c b/drivers/ub/cdma/cdma_tp.c new file mode 100644 index 0000000000000000000000000000000000000000..a77f1164b416983a5461d79acabcda4903851ca9 --- /dev/null +++ b/drivers/ub/cdma/cdma_tp.c @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "CDMA: " fmt + +#include "cdma_common.h" +#include "cdma_mbox.h" +#include "cdma_tp.h" +#include + +static inline int cdma_ctrlq_msg_send(struct cdma_dev *cdev, + struct ubase_ctrlq_msg *msg) +{ + int ret; + + ret = ubase_ctrlq_send_msg(cdev->adev, msg); + if (ret) + dev_err(cdev->dev, "ctrlq send msg failed, ret = %d.\n", ret); + + return ret; +} + +static int cdma_ctrlq_create_ctp(struct cdma_dev *cdev, + struct cdma_tp_cfg *cfg, u32 *tpn) +{ + struct cdma_ctrlq_tp_create_cfg ctrlq_tp; + struct cdma_ctrlq_tp_ret tp_out = { 0 }; + struct ubase_ctrlq_msg msg = { 0 }; + int ret; + + ctrlq_tp = (struct cdma_ctrlq_tp_create_cfg) { + .seid_flag = CDMA_CTRLQ_FLAG_ON, + .deid_flag = CDMA_CTRLQ_FLAG_ON, + .scna = cfg->scna, + .dcna = cfg->dcna, + .seid[0] = cfg->seid, + .deid[0] = cfg->deid, + .route_type = CDMA_ROUTE_TYPE_CNA, + .trans_type = CDMA_TRANS_TYPE_CDMA_CTP + }; + + msg = (struct ubase_ctrlq_msg) { + .service_ver = UBASE_CTRLQ_SER_VER_01, + .service_type = UBASE_CTRLQ_SER_TYPE_TP_ACL, + .opcode = CDMA_CTRLQ_CREATE_CTP, + .need_resp = CDMA_CTRLQ_FLAG_ON, + .is_resp = CDMA_CTRLQ_FLAG_OFF, + .in_size = sizeof(ctrlq_tp), + .in = &ctrlq_tp, + .out_size = sizeof(tp_out), + .out = &tp_out + }; + + ret = cdma_ctrlq_msg_send(cdev, &msg); + if (ret) + return ret; + + ret = tp_out.ret; + if (ret <= 0) { + dev_err(cdev->dev, + "create ctp failed, scna = %u, dcna = %u, ret = %d.\n", + ctrlq_tp.scna, ctrlq_tp.dcna, ret); + return -EFAULT; + } + *tpn = ret & CDMA_TPN_MASK; + + return 0; +} + +static void cdma_ctrlq_delete_ctp(struct cdma_dev *cdev, u32 tpn, + struct cdma_tp_cfg *cfg) +{ + struct cdma_ctrlq_tp_delete_cfg ctrlq_tp = { 0 }; + struct cdma_ctrlq_tp_ret tp_out = { 0 }; + struct ubase_ctrlq_msg msg = { 0 }; + int ret; + + ctrlq_tp.seid_flag = CDMA_CTRLQ_FLAG_ON; + ctrlq_tp.deid_flag = CDMA_CTRLQ_FLAG_ON; + ctrlq_tp.scna = cfg->scna; + ctrlq_tp.dcna = cfg->dcna; + ctrlq_tp.seid[0] = cfg->seid; + ctrlq_tp.deid[0] = cfg->deid; + ctrlq_tp.tpn = tpn; + ctrlq_tp.route_type = CDMA_ROUTE_TYPE_CNA; + ctrlq_tp.trans_type = CDMA_TRANS_TYPE_CDMA_CTP; + + msg.service_ver = UBASE_CTRLQ_SER_VER_01; + msg.service_type = UBASE_CTRLQ_SER_TYPE_TP_ACL; + msg.opcode = CDMA_CTRLQ_DELETE_CTP; + msg.need_resp = CDMA_CTRLQ_FLAG_ON; + msg.is_resp = CDMA_CTRLQ_FLAG_OFF; + msg.in_size = sizeof(ctrlq_tp); + msg.in = &ctrlq_tp; + msg.out_size = sizeof(tp_out); + msg.out = &tp_out; + + ret = cdma_ctrlq_msg_send(cdev, &msg); + if (ret) + dev_err(cdev->dev, + "delete ctp failed, tpn = %u, dcna = %u, ret = %d.\n", + tpn, cfg->dcna, ret); +} + +static struct cdma_tp *cdma_id_find_ctp(struct cdma_dev *cdev, u32 id) +{ + struct cdma_tp *tp; + + spin_lock(&cdev->ctp_table.lock); + tp = idr_find(&cdev->ctp_table.idr_tbl.idr, id); + if (!tp) + dev_err(cdev->dev, + "get tp from table failed, id = %u.\n", id); + spin_unlock(&cdev->ctp_table.lock); + + return tp; +} + +static struct cdma_tp *cdma_tpn_find_ctp(struct cdma_dev *cdev, u32 tpn) +{ + struct cdma_tp *tmp; + int id; + + spin_lock(&cdev->ctp_table.lock); + idr_for_each_entry(&cdev->ctp_table.idr_tbl.idr, tmp, id) { + if (tmp && tmp->base.tpn == tpn) { + spin_unlock(&cdev->ctp_table.lock); + return tmp; + } + } + + spin_unlock(&cdev->ctp_table.lock); + return NULL; +} + +static int cdma_alloc_tp_id(struct cdma_dev *cdev, struct cdma_tp *tp) +{ + struct cdma_table *tp_tbl = &cdev->ctp_table; + int id; + + idr_preload(GFP_KERNEL); + spin_lock(&tp_tbl->lock); + id = idr_alloc(&tp_tbl->idr_tbl.idr, tp, tp_tbl->idr_tbl.min, + tp_tbl->idr_tbl.max, GFP_NOWAIT); + if (id < 0) + dev_err(cdev->dev, "cdma tp id alloc failed.\n"); + spin_unlock(&tp_tbl->lock); + idr_preload_end(); + + return id; +} + +struct cdma_base_tp *cdma_create_ctp(struct cdma_dev *cdev, + struct cdma_tp_cfg *cfg) +{ + struct cdma_tp *tp; + u32 tpn; + int ret; + + ret = cdma_ctrlq_create_ctp(cdev, cfg, &tpn); + if (ret) { + dev_err(cdev->dev, "get tp failed, ret = %d.\n", ret); + return NULL; + } + + tp = (struct cdma_tp *)cdma_tpn_find_ctp(cdev, tpn); + if (tp) { + refcount_inc(&tp->refcount); + return &tp->base; + } + + tp = kzalloc(sizeof(*tp), GFP_KERNEL); + if (!tp) + goto err_alloc_tp; + + refcount_set(&tp->refcount, 1); + tp->base.cfg = *cfg; + tp->base.tpn = tpn; + tp->dev = cdev; + + ret = cdma_alloc_tp_id(cdev, tp); + if (ret < 0) + goto err_alloc_tpid; + + tp->base.tp_id = ret; + refcount_inc(&tp->refcount); + + dev_dbg(cdev->dev, "create ctp id = %u, tpn = %u, seid = %u, dcna = %u\n", + tp->base.tp_id, tpn, cfg->seid, cfg->dcna); + + return &tp->base; + +err_alloc_tpid: + kfree(tp); +err_alloc_tp: + cdma_ctrlq_delete_ctp(cdev, tpn, cfg); + + return NULL; +} + +void cdma_delete_ctp(struct cdma_dev *cdev, u32 tp_id) +{ + struct cdma_tp_cfg cfg = { 0 }; + struct cdma_tp *tp; + bool flag = false; + u32 tpn; + + if (!cdev) + return; + + tp = cdma_id_find_ctp(cdev, tp_id); + if (!tp) + return; + + spin_lock(&cdev->ctp_table.lock); + refcount_dec(&tp->refcount); + if (refcount_dec_if_one(&tp->refcount)) { + if (cdev->status != CDMA_SUSPEND) { + flag = true; + tpn = tp->base.tpn; + cfg = tp->base.cfg; + } + + dev_dbg(cdev->dev, + "refcout of tp %u is equal to one and erased.\n", tp_id); + idr_remove(&cdev->ctp_table.idr_tbl.idr, tp_id); + kfree(tp); + } + spin_unlock(&cdev->ctp_table.lock); + + if (flag) + cdma_ctrlq_delete_ctp(cdev, tpn, &cfg); +} + +void cdma_destroy_ctp_imm(struct cdma_dev *cdev, u32 tp_id) +{ + struct cdma_tp_cfg cfg = { 0 }; + struct cdma_tp *tp; + u32 tpn; + + if (!cdev) + return; + + tp = cdma_id_find_ctp(cdev, tp_id); + if (!tp) + return; + + spin_lock(&cdev->ctp_table.lock); + tpn = tp->base.tpn; + cfg = tp->base.cfg; + idr_remove(&cdev->ctp_table.idr_tbl.idr, tp_id); + kfree(tp); + spin_unlock(&cdev->ctp_table.lock); + + cdma_ctrlq_delete_ctp(cdev, tpn, &cfg); +} diff --git a/drivers/ub/cdma/cdma_tp.h b/drivers/ub/cdma/cdma_tp.h new file mode 100644 index 0000000000000000000000000000000000000000..72019df35d74606c1c9904d07a82713316aecb05 --- /dev/null +++ b/drivers/ub/cdma/cdma_tp.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_TP_H__ +#define __CDMA_TP_H__ + +#include "cdma_types.h" + +#define CDMA_CTRLQ_FLAG_ON 1 +#define CDMA_CTRLQ_FLAG_OFF 0 +#define CDMA_TPN_MASK 0xffffff +#define CDMA_EID_DW_SIZE 4 + +struct cdma_tp { + struct cdma_dev *dev; + struct cdma_base_tp base; + refcount_t refcount; + struct completion ae_comp; +}; + +enum cdma_tp_ctrlq_cmd { + CDMA_CTRLQ_CREATE_CTP = 0x01, + CDMA_CTRLQ_DELETE_CTP = 0x02 +}; + +enum cdma_tp_route_type { + CDMA_ROUTE_TYPE_IPV4, + CDMA_ROUTE_TYPE_IPV6, + CDMA_ROUTE_TYPE_CNA, + CDMA_ROUTE_TYPE_MAX +}; + +enum cdma_tp_trans_type { + CDMA_TRANS_TYPE_URMA_TP, + CDMA_TRANS_TYPE_URMA_CTP, + CDMA_TRANS_TYPE_UMS_TP, + CDMA_TRANS_TYPE_CDMA_CTP, + CDMA_TRANS_TYPE_MAX +}; + +struct cdma_ctrlq_tp_create_cfg { + u32 seid_flag; /* 0: 128bits eid, 1: 20bits eid */ + u32 seid[CDMA_EID_DW_SIZE]; + u32 scna; + u32 deid_flag; + u32 deid[CDMA_EID_DW_SIZE]; + u32 dcna; + u32 route_type : 4; /* 0-IPv4, 1-IPv6, 2-CNA */ + u32 trans_type : 4; + u32 rsv : 24; +}; + +struct cdma_ctrlq_tp_ret { + int ret; +}; + +struct cdma_ctrlq_tp_delete_cfg { + u32 seid_flag; + u32 seid[CDMA_EID_DW_SIZE]; + u32 scna; + u32 deid_flag; + u32 deid[CDMA_EID_DW_SIZE]; + u32 dcna; + u32 route_type : 4; /* 0-IPv4, 1-IPv6, 2-CNA */ + u32 trans_type : 4; + u32 rsv : 24; + u32 tpn; +}; + +struct cdma_base_tp *cdma_create_ctp(struct cdma_dev *cdev, + struct cdma_tp_cfg *cfg); + +void cdma_delete_ctp(struct cdma_dev *cdev, uint32_t tp_id); + +void cdma_destroy_ctp_imm(struct cdma_dev *cdev, uint32_t tp_id); +#endif /* CDMA_TP_H */ diff --git a/drivers/ub/cdma/cdma_types.h b/drivers/ub/cdma/cdma_types.h new file mode 100644 index 0000000000000000000000000000000000000000..947c360ba2ef470c35b9584710a9c27bfdc5fe82 --- /dev/null +++ b/drivers/ub/cdma/cdma_types.h @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_TYPES_H__ +#define __CDMA_TYPES_H__ + +#include +#include +#include +#include "cdma.h" + +enum cdma_event_type { + CDMA_EVENT_JFC_ERR, + CDMA_EVENT_JFS_ERR, + CDMA_EVENT_DEV_INVALID, +}; + +enum cdma_remove_reason { + /* Context deletion. This call should delete the actual object itself */ + CDMA_REMOVE_CLOSE, + /* Driver is being hot-unplugged. This call should delete the actual object itself */ + CDMA_REMOVE_DRIVER_REMOVE, +}; + +struct cdma_ucontext { + struct cdma_dev *dev; + u32 eid; + u32 eid_index; +}; + +struct cdma_udrv_priv { + u64 in_addr; + u32 in_len; + u64 out_addr; + u32 out_len; +}; + +union cdma_jfs_flag { + struct { + u32 error_suspend : 1; + u32 outorder_comp : 1; + u32 reserved : 30; + } bs; + u32 value; +}; + +struct cdma_jfs_cfg { + u32 depth; + union cdma_jfs_flag flag; + u32 eid_index; + u8 priority; + u8 max_sge; + u8 max_rsge; + u8 rnr_retry; + u8 err_timeout; + u32 jfc_id; + u32 sqe_pos; + u32 rmt_eid; + u32 tpn; + u32 pld_pos; + u32 pld_token_id; + u32 queue_id; + u32 trans_mode; +}; + +struct cdma_tp_cfg { + u32 scna; + u32 dcna; + u32 seid; + u32 deid; +}; + +struct cdma_base_tp { + struct cdma_ucontext *uctx; + struct cdma_tp_cfg cfg; + u64 usr_tp; + u32 tpn; + u32 tp_id; +}; + +struct cdma_udata { + struct cdma_context *uctx; + struct cdma_udrv_priv *udrv_data; +}; + +struct cdma_event { + struct cdma_dev *dev; + union { + struct cdma_base_jfc *jfc; + struct cdma_base_jfs *jfs; + u32 eid_idx; + } element; + enum cdma_event_type event_type; +}; + +typedef void (*cdma_event_callback_t)(struct cdma_event *event, + struct cdma_context *ctx); + +struct cdma_base_jfs { + struct cdma_dev *dev; + struct cdma_context *ctx; + struct cdma_jfs_cfg cfg; + cdma_event_callback_t jfae_handler; + u64 usr_jfs; + u32 id; + atomic_t use_cnt; + struct cdma_jfs_event jfs_event; +}; + +struct cdma_jfc_cfg { + u32 depth; + u32 ceqn; + u32 queue_id; +}; + +struct cdma_base_jfc; + +typedef void (*cdma_comp_callback_t)(struct cdma_base_jfc *jfc); + +struct cdma_base_jfc { + struct cdma_dev *dev; + struct cdma_context *ctx; + struct cdma_jfc_cfg jfc_cfg; + u32 id; + cdma_comp_callback_t jfce_handler; + cdma_event_callback_t jfae_handler; + struct hlist_node hnode; + atomic_t use_cnt; + struct cdma_jfc_event jfc_event; +}; + +enum cdma_wr_opcode { + CDMA_WR_OPC_WRITE = 0x00, + CDMA_WR_OPC_WRITE_NOTIFY = 0x02, + CDMA_WR_OPC_READ = 0x10, + CDMA_WR_OPC_CAS = 0x20, + CDMA_WR_OPC_FADD = 0x22, + CDMA_WR_OPC_LAST +}; + +struct cdma_mn { + struct mmu_notifier mn; + struct mm_struct *mm; +}; + +struct cdma_file { + struct cdma_dev *cdev; + struct list_head list; + struct mutex ctx_mutex; + struct cdma_context *uctx; + struct idr idr; + spinlock_t idr_lock; + struct mutex umap_mutex; + struct list_head umaps_list; + struct page *fault_page; + struct cdma_mn mn_notifier; + struct kref ref; +}; + +struct cdma_umap_priv { + struct vm_area_struct *vma; + struct list_head node; +}; + +#endif diff --git a/drivers/ub/cdma/cdma_uobj.c b/drivers/ub/cdma/cdma_uobj.c new file mode 100644 index 0000000000000000000000000000000000000000..92fe4da441ea623889bf6f7270a1b8d26242be6b --- /dev/null +++ b/drivers/ub/cdma/cdma_uobj.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#include +#include "cdma_mmap.h" +#include "cdma_uobj.h" +#include "cdma_chardev.h" + +static int cdma_uobj_alloc_idr(struct cdma_uobj *uobj) +{ + int ret; + + idr_preload(GFP_KERNEL); + spin_lock(&uobj->cfile->idr_lock); + + ret = idr_alloc(&uobj->cfile->idr, uobj, 0, U32_MAX, GFP_NOWAIT); + if (ret >= 0) + uobj->id = ret; + + spin_unlock(&uobj->cfile->idr_lock); + idr_preload_end(); + + return ret < 0 ? ret : 0; +} + +static inline void cdma_uobj_remove_idr(struct cdma_uobj *uobj) +{ + spin_lock(&uobj->cfile->idr_lock); + idr_remove(&uobj->cfile->idr, uobj->id); + spin_unlock(&uobj->cfile->idr_lock); +} + +static struct cdma_uobj *cdma_uobj_alloc(struct cdma_file *cfile, + enum UOBJ_TYPE obj_type) +{ + struct cdma_uobj *uobj; + + uobj = kzalloc(sizeof(*uobj), GFP_KERNEL); + if (uobj == NULL) + return ERR_PTR(-ENOMEM); + + atomic_set(&uobj->rcnt, 0); + uobj->cfile = cfile; + uobj->type = obj_type; + + return uobj; +} + +static inline void cdma_uobj_free(struct cdma_uobj *uobj) +{ + kfree(uobj); +} + +static inline void cdma_uobj_remove(struct cdma_uobj *uobj) +{ + idr_remove(&uobj->cfile->idr, uobj->id); + cdma_uobj_free(uobj); +} + +void cdma_init_uobj_idr(struct cdma_file *cfile) +{ + idr_init(&cfile->idr); + spin_lock_init(&cfile->idr_lock); +} + +struct cdma_uobj *cdma_uobj_create(struct cdma_file *cfile, + enum UOBJ_TYPE obj_type) +{ + struct cdma_uobj *uobj; + int ret; + + uobj = cdma_uobj_alloc(cfile, obj_type); + if (IS_ERR(uobj)) + return uobj; + + ret = cdma_uobj_alloc_idr(uobj); + if (ret) + goto err_free_uobj; + + return uobj; + +err_free_uobj: + cdma_uobj_free(uobj); + + return ERR_PTR(ret); +} + +void cdma_uobj_delete(struct cdma_uobj *uobj) +{ + cdma_uobj_remove_idr(uobj); + cdma_uobj_free(uobj); +} + +struct cdma_uobj *cdma_uobj_get(struct cdma_file *cfile, int id, + enum UOBJ_TYPE obj_type) +{ + struct cdma_uobj *uobj; + + spin_lock(&cfile->idr_lock); + uobj = idr_find(&cfile->idr, id); + if (uobj == NULL || uobj->type != obj_type) + uobj = ERR_PTR(-ENOENT); + spin_unlock(&cfile->idr_lock); + + return uobj; +} + +void cdma_cleanup_context_uobj(struct cdma_file *cfile, enum cdma_remove_reason why) +{ + struct cdma_uobj *uobj; + int id; + + if (why == CDMA_REMOVE_DRIVER_REMOVE) + cdma_unmap_vma_pages(cfile); + + spin_lock(&cfile->idr_lock); + idr_for_each_entry(&cfile->idr, uobj, id) + cdma_uobj_remove(uobj); + spin_unlock(&cfile->idr_lock); +} + +void cdma_close_uobj_fd(struct cdma_file *cfile) +{ + kref_put(&cfile->ref, cdma_release_file); +} diff --git a/drivers/ub/cdma/cdma_uobj.h b/drivers/ub/cdma/cdma_uobj.h new file mode 100644 index 0000000000000000000000000000000000000000..f343559a33ce6e77f2e0be6728ac425f0abb04aa --- /dev/null +++ b/drivers/ub/cdma/cdma_uobj.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef __CDMA_UOBJ_H__ +#define __CDMA_UOBJ_H__ +#include "cdma_types.h" + +enum UOBJ_TYPE { + UOBJ_TYPE_JFCE, + UOBJ_TYPE_JFC, + UOBJ_TYPE_CTP, + UOBJ_TYPE_JFS, + UOBJ_TYPE_QUEUE, + UOBJ_TYPE_SEGMENT +}; + +struct cdma_uobj { + struct cdma_file *cfile; + enum UOBJ_TYPE type; + int id; + void *object; + atomic_t rcnt; +}; + +void cdma_init_uobj_idr(struct cdma_file *cfile); +struct cdma_uobj *cdma_uobj_create(struct cdma_file *cfile, + enum UOBJ_TYPE obj_type); +void cdma_uobj_delete(struct cdma_uobj *uobj); +struct cdma_uobj *cdma_uobj_get(struct cdma_file *cfile, int id, + enum UOBJ_TYPE type); +void cdma_cleanup_context_uobj(struct cdma_file *cfile, enum cdma_remove_reason why); +void cdma_close_uobj_fd(struct cdma_file *cfile); + +#endif diff --git a/drivers/ub/ubase/debugfs/ubase_ctx_debugfs.c b/drivers/ub/ubase/debugfs/ubase_ctx_debugfs.c index 1004033c1581da63d9bd5486065987ba431b06e4..63f27093aec12985340dfe01b53b1860455bc91e 100644 --- a/drivers/ub/ubase/debugfs/ubase_ctx_debugfs.c +++ b/drivers/ub/ubase/debugfs/ubase_ctx_debugfs.c @@ -7,6 +7,237 @@ #include #include "ubase_debugfs.h" +#include "ubase_hw.h" +#include "ubase_mailbox.h" +#include "ubase_tp.h" +#include "ubase_ctx_debugfs.h" + +#define UBASE_DEFAULT_CTXGN 0 + +static void ubase_dump_eq_ctx(struct seq_file *s, struct ubase_eq *eq) +{ + seq_printf(s, "%-5u", eq->eqn); + seq_printf(s, "%-13u", eq->entries_num); + seq_printf(s, "%-7u", eq->state); + seq_printf(s, "%-8u", eq->arm_st); + seq_printf(s, "%-10u", eq->eqe_size); + seq_printf(s, "%-11u", eq->eq_period); + seq_printf(s, "%-14u", eq->coalesce_cnt); + seq_printf(s, "%-6u", eq->irqn); + seq_printf(s, "%-10u", eq->eqc_irqn); + seq_printf(s, "%-12u", eq->cons_index); + seq_puts(s, "\n"); +} + +static void ubase_eq_ctx_titles_print(struct seq_file *s) +{ + seq_puts(s, "EQN ENTRIES_NUM STATE ARM_ST EQE_SIZE EQ_PERIOD "); + seq_puts(s, "COALESCE_CNT IRQN EQC_IRQN CONS_INDEX\n"); +} + +static void ubase_dump_aeq_ctx(struct seq_file *s, struct ubase_dev *udev, u32 idx) +{ + struct ubase_aeq *aeq = &udev->irq_table.aeq; + struct ubase_eq *eq = &aeq->eq; + + ubase_dump_eq_ctx(s, eq); +} + +static void ubase_dump_ceq_ctx(struct seq_file *s, struct ubase_dev *udev, u32 idx) +{ + struct ubase_ceq *ceq = &udev->irq_table.ceqs.ceq[idx]; + struct ubase_eq *eq = &ceq->eq; + + ubase_dump_eq_ctx(s, eq); +} + +static void ubase_tpg_ctx_titles_print(struct seq_file *s) +{ + seq_puts(s, "CHANNEL_ID TPGN TP_SHIFT VALID_TP "); + seq_puts(s, "START_TPN TPG_STATE TP_CNT\n"); +} + +static void ubase_dump_tpg_ctx(struct seq_file *s, struct ubase_dev *udev, u32 idx) +{ + struct ubase_tpg *tpg = &udev->tp_ctx.tpg[idx]; + + seq_printf(s, "%-12u", idx); + seq_printf(s, "%-9u", tpg->mb_tpgn); + seq_printf(s, "%-10u", tpg->tp_shift); + seq_printf(s, "%-10lu", tpg->valid_tp); + seq_printf(s, "%-11u", tpg->start_tpn); + seq_printf(s, "%-11u", tpg->tpg_state); + seq_printf(s, "%-8u", tpg->tp_cnt); + seq_puts(s, "\n"); +} + +enum ubase_dbg_ctx_type { + UBASE_DBG_AEQ_CTX = 0, + UBASE_DBG_CEQ_CTX, + UBASE_DBG_TPG_CTX, + UBASE_DBG_TP_CTX, +}; + +static u32 ubase_get_ctx_num(struct ubase_dev *udev, + enum ubase_dbg_ctx_type ctx_type, u32 ctxgn) +{ + struct ubase_adev_caps *unic_caps = &udev->caps.unic_caps; + u32 ctx_num = 0; + + switch (ctx_type) { + case UBASE_DBG_AEQ_CTX: + ctx_num = udev->caps.dev_caps.num_aeq_vectors; + break; + case UBASE_DBG_CEQ_CTX: + ctx_num = udev->irq_table.ceqs.num; + break; + case UBASE_DBG_TPG_CTX: + ctx_num = unic_caps->tpg.max_cnt; + break; + case UBASE_DBG_TP_CTX: + spin_lock(&udev->tp_ctx.tpg_lock); + if (udev->tp_ctx.tpg) + ctx_num = udev->tp_ctx.tpg[ctxgn].tp_cnt; + spin_unlock(&udev->tp_ctx.tpg_lock); + break; + default: + ubase_err(udev, "failed to get ctx num, ctx_type = %u.\n", + ctx_type); + break; + } + + return ctx_num; +} + +static int ubase_dbg_dump_context(struct seq_file *s, + enum ubase_dbg_ctx_type ctx_type) +{ + struct ubase_dbg_ctx { + void (*print_ctx_titles)(struct seq_file *s); + void (*get_ctx)(struct seq_file *s, struct ubase_dev *udev, u32 idx); + } dbg_ctx[] = { + {ubase_eq_ctx_titles_print, ubase_dump_aeq_ctx}, + {ubase_eq_ctx_titles_print, ubase_dump_ceq_ctx}, + {ubase_tpg_ctx_titles_print, ubase_dump_tpg_ctx}, + }; + struct ubase_dev *udev = dev_get_drvdata(s->private); + struct ubase_adev_caps *unic_caps = &udev->caps.unic_caps; + unsigned long port_bitmap; + u32 tp_pos, i; + + dbg_ctx[ctx_type].print_ctx_titles(s); + + port_bitmap = unic_caps->utp_port_bitmap; + for (i = 0; i < ubase_get_ctx_num(udev, ctx_type, UBASE_DEFAULT_CTXGN); i++) { + if (ctx_type != UBASE_DBG_TP_CTX) { + dbg_ctx[ctx_type].get_ctx(s, udev, i); + continue; + } + + tp_pos = (i % unic_caps->tpg.depth) * UBASE_TP_PORT_BITMAP_STEP; + if (test_bit(tp_pos, &port_bitmap)) + dbg_ctx[ctx_type].get_ctx(s, udev, i); + } + + return 0; +} + +struct ubase_ctx_info { + u32 start_idx; + u32 ctx_size; + u8 op; + const char *ctx_name; +}; + +static inline u32 ubase_get_ctx_group_num(struct ubase_dev *udev, + enum ubase_dbg_ctx_type ctx_type) +{ + if (ctx_type == UBASE_DBG_TP_CTX) + return udev->caps.unic_caps.tpg.max_cnt; + + return 1; +} + +static void ubase_get_ctx_info(struct ubase_dev *udev, + enum ubase_dbg_ctx_type ctx_type, + struct ubase_ctx_info *ctx_info, u32 ctxgn) +{ + switch (ctx_type) { + case UBASE_DBG_AEQ_CTX: + ctx_info->start_idx = 0; + ctx_info->ctx_size = UBASE_AEQ_CTX_SIZE; + ctx_info->op = UBASE_MB_QUERY_AEQ_CONTEXT; + ctx_info->ctx_name = "aeq"; + break; + case UBASE_DBG_CEQ_CTX: + ctx_info->start_idx = 0; + ctx_info->ctx_size = UBASE_CEQ_CTX_SIZE; + ctx_info->op = UBASE_MB_QUERY_CEQ_CONTEXT; + ctx_info->ctx_name = "ceq"; + break; + case UBASE_DBG_TPG_CTX: + ctx_info->start_idx = udev->caps.unic_caps.tpg.start_idx; + ctx_info->ctx_size = udev->ctx_buf.tpg.entry_size; + ctx_info->op = UBASE_MB_QUERY_TPG_CONTEXT; + ctx_info->ctx_name = "tpg"; + break; + case UBASE_DBG_TP_CTX: + spin_lock(&udev->tp_ctx.tpg_lock); + ctx_info->start_idx = udev->tp_ctx.tpg ? + udev->tp_ctx.tpg[ctxgn].start_tpn : 0; + spin_unlock(&udev->tp_ctx.tpg_lock); + + ctx_info->ctx_size = udev->ctx_buf.tp.entry_size; + ctx_info->op = UBASE_MB_QUERY_TP_CONTEXT; + ctx_info->ctx_name = "tp"; + break; + default: + ubase_err(udev, "failed to get ctx info, ctx_type = %u.\n", + ctx_type); + break; + } +} + +static void ubase_mask_eq_ctx_key_words(void *buf) +{ + struct ubase_eq_ctx *eq = (struct ubase_eq_ctx *)buf; + + eq->eqe_base_addr_l = 0; + eq->eqe_base_addr_h = 0; + eq->eqe_token_id = 0; + eq->eqe_token_value = 0; +} + +static void ubase_mask_tp_ctx_key_words(void *buf) +{ + struct ubase_tp_ctx *tp = (struct ubase_tp_ctx *)buf; + + tp->wqe_ba_l = 0; + tp->wqe_ba_h = 0; + tp->tp_wqe_token_id = 0; + tp->reorder_q_addr_l = 0; + tp->reorder_q_addr_h = 0; + tp->scc_token = 0; + tp->scc_token_1 = 0; +} + +static void ubase_mask_ctx_key_words(void *buf, + enum ubase_dbg_ctx_type ctx_type) +{ + switch (ctx_type) { + case UBASE_DBG_AEQ_CTX: + case UBASE_DBG_CEQ_CTX: + ubase_mask_eq_ctx_key_words(buf); + break; + case UBASE_DBG_TPG_CTX: + break; + case UBASE_DBG_TP_CTX: + ubase_mask_tp_ctx_key_words(buf); + break; + default: + break; + } +} static void __ubase_print_context_hw(struct seq_file *s, void *ctx_addr, u32 ctx_len) @@ -21,6 +252,19 @@ static void __ubase_print_context_hw(struct seq_file *s, void *ctx_addr, } } +/** + * ubase_print_context_hw() - formatted the context output to seq file + * @s: seq_file + * @ctx_addr: context address + * @ctx_len: context length + * + * This function outputs the contents of `ctx_addr` to a seq_file according to + * the specified format. + * Each line in the file is 32 bits, and the number of lines is `ctx_len / sizeof(u32)`. + * If `ctx_len` is not an integer multiple of 4, there will be truncation at the end. + * + * Context: Any context. + */ void ubase_print_context_hw(struct seq_file *s, void *ctx_addr, u32 ctx_len) { if (!s || !ctx_addr) @@ -29,3 +273,141 @@ void ubase_print_context_hw(struct seq_file *s, void *ctx_addr, u32 ctx_len) __ubase_print_context_hw(s, ctx_addr, ctx_len); } EXPORT_SYMBOL(ubase_print_context_hw); + +static int ubase_dbg_dump_ctx_hw(struct seq_file *s, void *data, + enum ubase_dbg_ctx_type ctx_type) +{ + struct ubase_dev *udev = dev_get_drvdata(s->private); + struct ubase_ctx_info ctx_info = {0}; + struct ubase_cmd_mailbox *mailbox; + u32 max_ctxgn, ctxn, ctxgn; + struct ubase_mbx_attr attr; + int ret = 0; + + if (!test_bit(UBASE_STATE_INITED_B, &udev->state_bits) || + test_bit(UBASE_STATE_RST_HANDLING_B, &udev->state_bits)) + return -EBUSY; + + mailbox = __ubase_alloc_cmd_mailbox(udev); + if (IS_ERR_OR_NULL(mailbox)) { + ubase_err(udev, + "failed to alloc mailbox for dump hw context.\n"); + return -ENOMEM; + } + + max_ctxgn = ubase_get_ctx_group_num(udev, ctx_type); + for (ctxgn = 0; ctxgn < max_ctxgn; ctxgn++) { + ubase_get_ctx_info(udev, ctx_type, &ctx_info, ctxgn); + + for (ctxn = 0; ctxn < ubase_get_ctx_num(udev, ctx_type, ctxgn); ctxn++) { + ubase_fill_mbx_attr(&attr, ctxn + ctx_info.start_idx, + ctx_info.op, 0); + ret = __ubase_hw_upgrade_ctx_ex(udev, &attr, mailbox); + if (ret) { + ubase_err(udev, + "failed to post query %s ctx mbx, ret = %d.\n", + ctx_info.ctx_name, ret); + goto upgrade_ctx_err; + } + + seq_printf(s, "offset\t%s%u\n", ctx_info.ctx_name, + ctxn + ctx_info.start_idx); + ubase_mask_ctx_key_words(mailbox->buf, ctx_type); + __ubase_print_context_hw(s, mailbox->buf, ctx_info.ctx_size); + seq_puts(s, "\n"); + } + } + +upgrade_ctx_err: + __ubase_free_cmd_mailbox(udev, mailbox); + + return ret; +} + +int ubase_dbg_dump_aeq_context(struct seq_file *s, void *data) +{ + return ubase_dbg_dump_context(s, UBASE_DBG_AEQ_CTX); +} + +int ubase_dbg_dump_ceq_context(struct seq_file *s, void *data) +{ + struct ubase_dev *udev = dev_get_drvdata(s->private); + int ret; + + if (!mutex_trylock(&udev->irq_table.ceq_lock)) + return -EBUSY; + + if (!udev->irq_table.ceqs.ceq) { + mutex_unlock(&udev->irq_table.ceq_lock); + return -EBUSY; + } + + ret = ubase_dbg_dump_context(s, UBASE_DBG_CEQ_CTX); + mutex_unlock(&udev->irq_table.ceq_lock); + + return ret; +} + +int ubase_dbg_dump_tpg_ctx(struct seq_file *s, void *data) +{ + struct ubase_dev *udev = dev_get_drvdata(s->private); + int ret; + + if (!test_bit(UBASE_STATE_INITED_B, &udev->state_bits)) + return -EBUSY; + + if (!ubase_get_ctx_num(udev, UBASE_DBG_TPG_CTX, UBASE_DEFAULT_CTXGN)) + return -EOPNOTSUPP; + + if (!spin_trylock(&udev->tp_ctx.tpg_lock)) + return -EBUSY; + + if (!udev->tp_ctx.tpg) { + spin_unlock(&udev->tp_ctx.tpg_lock); + return -EBUSY; + } + + ret = ubase_dbg_dump_context(s, UBASE_DBG_TPG_CTX); + spin_unlock(&udev->tp_ctx.tpg_lock); + + return ret; +} + +int ubase_dbg_dump_tpg_ctx_hw(struct seq_file *s, void *data) +{ + struct ubase_dev *udev = dev_get_drvdata(s->private); + + if (!test_bit(UBASE_STATE_INITED_B, &udev->state_bits)) + return -EBUSY; + + if (!ubase_get_ctx_num(udev, UBASE_DBG_TPG_CTX, UBASE_DEFAULT_CTXGN)) + return -EOPNOTSUPP; + + return ubase_dbg_dump_ctx_hw(s, data, UBASE_DBG_TPG_CTX); +} + +int ubase_dbg_dump_tp_ctx_hw(struct seq_file *s, void *data) +{ + struct ubase_dev *udev = dev_get_drvdata(s->private); + + if (!test_bit(UBASE_STATE_INITED_B, &udev->state_bits)) + return -EBUSY; + + if (!ubase_get_ctx_num(udev, UBASE_DBG_TP_CTX, UBASE_DEFAULT_CTXGN)) + return -EOPNOTSUPP; + + if (!ubase_get_ctx_group_num(udev, UBASE_DBG_TP_CTX)) + return -EOPNOTSUPP; + + return ubase_dbg_dump_ctx_hw(s, data, UBASE_DBG_TP_CTX); +} + +int ubase_dbg_dump_aeq_ctx_hw(struct seq_file *s, void *data) +{ + return ubase_dbg_dump_ctx_hw(s, data, UBASE_DBG_AEQ_CTX); +} + +int ubase_dbg_dump_ceq_ctx_hw(struct seq_file *s, void *data) +{ + return ubase_dbg_dump_ctx_hw(s, data, UBASE_DBG_CEQ_CTX); +} diff --git a/drivers/ub/ubase/debugfs/ubase_ctx_debugfs.h b/drivers/ub/ubase/debugfs/ubase_ctx_debugfs.h new file mode 100644 index 0000000000000000000000000000000000000000..532665141fc8d3e8d40b433224e3d70e39269ed1 --- /dev/null +++ b/drivers/ub/ubase/debugfs/ubase_ctx_debugfs.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. + * + */ + +#ifndef __UBASE_CTX_DEBUGFS_H__ +#define __UBASE_CTX_DEBUGFS_H__ + +struct device; + +int ubase_dbg_dump_aeq_context(struct seq_file *s, void *data); +int ubase_dbg_dump_ceq_context(struct seq_file *s, void *data); +int ubase_dbg_dump_tpg_ctx(struct seq_file *s, void *data); +int ubase_dbg_dump_tp_ctx_hw(struct seq_file *s, void *data); +int ubase_dbg_dump_tpg_ctx_hw(struct seq_file *s, void *data); +int ubase_dbg_dump_aeq_ctx_hw(struct seq_file *s, void *data); +int ubase_dbg_dump_ceq_ctx_hw(struct seq_file *s, void *data); + +#endif diff --git a/drivers/ub/ubase/debugfs/ubase_debugfs.c b/drivers/ub/ubase/debugfs/ubase_debugfs.c index d7495d8b4ef7297eccc5e5b58d0a24f9a3d4a4f7..9476cf17fa013d779f9cdbe7b1cd35616a7e41db 100644 --- a/drivers/ub/ubase/debugfs/ubase_debugfs.c +++ b/drivers/ub/ubase/debugfs/ubase_debugfs.c @@ -8,6 +8,8 @@ #include #include +#include "ubase_cmd.h" +#include "ubase_ctx_debugfs.h" #include "ubase_dev.h" #include "ubase_hw.h" #include "ubase_qos_debugfs.h" @@ -22,6 +24,7 @@ static int ubase_dbg_dump_rst_info(struct seq_file *s, void *data) seq_printf(s, "ELR reset count: %u\n", udev->reset_stat.elr_reset_cnt); seq_printf(s, "port reset count: %u\n", udev->reset_stat.port_reset_cnt); + seq_printf(s, "himac reset count: %u\n", udev->reset_stat.himac_reset_cnt); seq_printf(s, "reset done count: %u\n", udev->reset_stat.reset_done_cnt); seq_printf(s, "HW reset done count: %u\n", udev->reset_stat.hw_reset_done_cnt); seq_printf(s, "reset fail count: %u\n", udev->reset_stat.reset_fail_cnt); @@ -30,6 +33,274 @@ static int ubase_dbg_dump_rst_info(struct seq_file *s, void *data) return 0; } +static void ubase_dbg_dump_caps_bits(struct seq_file *s, struct ubase_dev *udev) +{ +#define CAP_FMT(name) "\tsupport_" #name ": %d\n" +#define PTRINT_CAP(name, func) seq_printf(s, CAP_FMT(name), func(udev)) + + PTRINT_CAP(ub_link, ubase_dev_ubl_supported); + PTRINT_CAP(ta_extdb_buffer_config, ubase_dev_ta_extdb_buf_supported); + PTRINT_CAP(ta_timer_buffer_config, ubase_dev_ta_timer_buf_supported); + PTRINT_CAP(err_handle, ubase_dev_err_handle_supported); + PTRINT_CAP(ctrlq, ubase_dev_ctrlq_supported); + PTRINT_CAP(eth_mac, ubase_dev_eth_mac_supported); + PTRINT_CAP(mac_stats, ubase_dev_mac_stats_supported); + PTRINT_CAP(prealloc, __ubase_dev_prealloc_supported); + PTRINT_CAP(udma, ubase_dev_udma_supported); + PTRINT_CAP(unic, ubase_dev_unic_supported); + PTRINT_CAP(uvb, ubase_dev_uvb_supported); + PTRINT_CAP(ip_over_urma, ubase_ip_over_urma_supported); + if (ubase_ip_over_urma_supported(udev)) + PTRINT_CAP(ip_over_urma_utp, ubase_ip_over_urma_utp_supported); + PTRINT_CAP(activate_proxy, ubase_activate_proxy_supported); + PTRINT_CAP(utp, ubase_utp_supported); +} + +static void ubase_dbg_dump_caps_info(struct seq_file *s, struct ubase_dev *udev) +{ + struct ubase_caps *dev_caps = &udev->caps.dev_caps; + struct ubase_dbg_common_caps_info { + const char *format; + u64 caps_info; + } ubase_common_caps_info[] = { + {"\tnum_ceq_vectors: %u\n", dev_caps->num_ceq_vectors}, + {"\tnum_aeq_vectors: %u\n", dev_caps->num_aeq_vectors}, + {"\tnum_misc_vectors: %u\n", dev_caps->num_misc_vectors}, + {"\taeqe_size: %u\n", dev_caps->aeqe_size}, + {"\tceqe_size: %u\n", dev_caps->ceqe_size}, + {"\taeqe_depth: %u\n", dev_caps->aeqe_depth}, + {"\tceqe_depth: %u\n", dev_caps->ceqe_depth}, + {"\ttotal_ue_num: %u\n", dev_caps->total_ue_num}, + {"\tta_extdb_buf_size: %llu\n", udev->ta_ctx.extdb_buf.size}, + {"\tta_timer_buf_size: %llu\n", udev->ta_ctx.timer_buf.size}, + {"\tpublic_jetty_cnt: %u\n", dev_caps->public_jetty_cnt}, + {"\tvl_num: %hhu\n", dev_caps->vl_num}, + {"\trsvd_jetty_cnt: %hu\n", dev_caps->rsvd_jetty_cnt}, + {"\tpacket_pattern_mode: %u\n", dev_caps->packet_pattern_mode}, + {"\tack_queue_num: %u\n", dev_caps->ack_queue_num}, + {"\toor_en: %u\n", dev_caps->oor_en}, + {"\treorder_queue_en: %u\n", dev_caps->reorder_queue_en}, + {"\ton_flight_size: %u\n", dev_caps->on_flight_size}, + {"\treorder_cap: %u\n", dev_caps->reorder_cap}, + {"\treorder_queue_shift: %u\n", dev_caps->reorder_queue_shift}, + {"\tat_times: %u\n", dev_caps->at_times}, + {"\tue_num: %u\n", dev_caps->ue_num}, + {"\tmac_stats_num: %u\n", dev_caps->mac_stats_num}, + {"\tlogic_port_bitmap: 0x%x\n", dev_caps->logic_port_bitmap}, + {"\tub_port_logic_id: %u\n", dev_caps->ub_port_logic_id}, + {"\tio_port_logic_id: %u\n", dev_caps->io_port_logic_id}, + {"\tio_port_id: %u\n", dev_caps->io_port_id}, + {"\tnl_port_id: %u\n", dev_caps->nl_port_id}, + {"\tchip_id: %u\n", dev_caps->chip_id}, + {"\tdie_id: %u\n", dev_caps->die_id}, + {"\tue_id: %u\n", dev_caps->ue_id}, + {"\tnl_id: %u\n", dev_caps->nl_id}, + {"\tfw_version: %u\n", dev_caps->fw_version}, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(ubase_common_caps_info); i++) + seq_printf(s, ubase_common_caps_info[i].format, + ubase_common_caps_info[i].caps_info); +} + +static void ubase_dbg_dump_common_caps(struct seq_file *s, struct ubase_dev *udev) +{ + struct ubase_caps *dev_caps = &udev->caps.dev_caps; + + ubase_dbg_dump_caps_info(s, udev); + + seq_puts(s, "\treq_vl:"); + ubase_dbg_dump_arr_info(s, dev_caps->req_vl, dev_caps->vl_num); + + seq_puts(s, "\tresp_vl:"); + ubase_dbg_dump_arr_info(s, dev_caps->resp_vl, dev_caps->vl_num); +} + +static void ubase_dbg_dump_adev_caps(struct seq_file *s, + struct ubase_adev_caps *caps) +{ + struct ubase_dbg_adev_caps_info { + const char *format; + u32 caps_info; + } ubase_adev_caps_info[] = { + {"\tjfs_max_cnt: %u\n", caps->jfs.max_cnt}, + {"\tjfs_depth: %u\n", caps->jfs.depth}, + {"\tjfr_max_cnt: %u\n", caps->jfr.max_cnt}, + {"\tjfr_depth: %u\n", caps->jfr.depth}, + {"\tjfc_max_cnt: %u\n", caps->jfc.max_cnt}, + {"\tjfc_depth: %u\n", caps->jfc.depth}, + {"\ttp_max_cnt: %u\n", caps->tp.max_cnt}, + {"\ttp_depth: %u\n", caps->tp.depth}, + {"\ttpg_max_cnt: %u\n", caps->tpg.max_cnt}, + {"\ttpg_depth: %u\n", caps->tpg.depth}, + {"\tcqe_size: %hu\n", caps->cqe_size}, + {"\tutp_port_bitmap: 0x%x\n", caps->utp_port_bitmap}, + {"\tjtg_max_cnt: %u\n", caps->jtg_max_cnt}, + {"\trc_max_cnt: %u\n", caps->rc_max_cnt}, + {"\trc_depth: %u\n", caps->rc_que_depth}, + {"\tprealloc_mem_dma_len: %llu\n", caps->pmem.dma_len}, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(ubase_adev_caps_info); i++) + seq_printf(s, ubase_adev_caps_info[i].format, + ubase_adev_caps_info[i].caps_info); +} + +static int ubase_dbg_dump_dev_caps(struct seq_file *s, void *data) +{ + struct ubase_dev *udev = dev_get_drvdata(s->private); + struct ubase_dev_caps *udev_caps = &udev->caps; + + seq_puts(s, "CAP_BITS:\n"); + ubase_dbg_dump_caps_bits(s, udev); + seq_puts(s, "\nCOMMON_CAPS:\n"); + ubase_dbg_dump_common_caps(s, udev); + + if (ubase_dev_pmu_supported(udev)) + return 0; + + seq_puts(s, "\nUNIC_CAPS:\n"); + ubase_dbg_dump_adev_caps(s, &udev_caps->unic_caps); + + if (ubase_dev_cdma_supported(udev)) + seq_puts(s, "\nCDMA_CAPS:\n"); + else + seq_puts(s, "\nUDMA_CAPS:\n"); + ubase_dbg_dump_adev_caps(s, &udev_caps->udma_caps); + + return 0; +} + +static int ubase_query_ubcl_config(struct ubase_dev *udev, u16 offset, + u16 is_query, u16 size, + struct ubase_ubcl_config_cmd *resp) +{ + struct ubase_ubcl_config_cmd req; + struct ubase_cmd_buf in, out; + int ret; + + memset(resp, 0, sizeof(*resp)); + memset(&req, 0, sizeof(req)); + req.offset = cpu_to_le16(offset); + req.size = cpu_to_le16(size); + req.is_query_size = cpu_to_le16(is_query); + + __ubase_fill_inout_buf(&in, UBASE_OPC_QUERY_UBCL_CONFIG, true, + sizeof(req), &req); + __ubase_fill_inout_buf(&out, UBASE_OPC_QUERY_UBCL_CONFIG, true, + sizeof(*resp), resp); + ret = __ubase_cmd_send_inout(udev, &in, &out); + if (ret && ret != -EPERM) + ubase_err(udev, "failed to query UBCL_config, ret = %d.\n", ret); + + if (ret == -EPERM) + return -EOPNOTSUPP; + + return ret; +} + +static void ubase_dbg_fill_ubcl_content(struct ubase_ubcl_config_cmd *resp, + u32 *addr, struct seq_file *s) +{ + int i, j; + + for (i = 0; i < UBASE_UBCL_CFG_DATA_NUM; i += UBASE_UBCL_CFG_DATA_ALIGN) { + seq_printf(s, "%08X: ", (*addr * UBASE_UBCL_CFG_DATA_ALIGN)); + for (j = 0; j < UBASE_UBCL_CFG_DATA_ALIGN; j++) + seq_printf(s, "%08X ", resp->data[i + j]); + seq_puts(s, "\n"); + + *addr += UBASE_UBCL_CFG_DATA_ALIGN; + if ((i * sizeof(u32)) >= resp->size) + break; + } +} + +static int ubase_dbg_dump_ubcl_config(struct seq_file *s, void *data) +{ + struct ubase_dev *udev = dev_get_drvdata(s->private); + struct ubase_ubcl_config_cmd resp = {0}; + u16 read_size = sizeof(resp.data); + u16 offset = 0; + u16 total_size; + u32 addr = 0; + int ret; + + if (!test_bit(UBASE_STATE_INITED_B, &udev->state_bits) || + test_bit(UBASE_STATE_RST_HANDLING_B, &udev->state_bits)) + return -EBUSY; + + ret = ubase_query_ubcl_config(udev, offset, 1, 0, &resp); + if (ret) + return ret; + total_size = le16_to_cpu(resp.size); + + seq_puts(s, "UBCL_config:\n"); + seq_printf(s, "total_size: %u\n", total_size); + while (offset < total_size) { + read_size = min(read_size, total_size - offset); + ret = ubase_query_ubcl_config(udev, offset, 0, read_size, &resp); + if (ret) + return ret; + offset += le16_to_cpu(resp.size); + + ubase_dbg_fill_ubcl_content(&resp, &addr, s); + } + + return 0; +} + +static int ubase_dbg_dump_activate_record(struct seq_file *s, void *data) +{ + struct ubase_dev *udev = dev_get_drvdata(s->private); + struct ubase_activate_dev_stats *record; + u8 cnt = 1, stats_cnt; + u64 total, idx; + + if (!test_bit(UBASE_STATE_INITED_B, &udev->state_bits) || + test_bit(UBASE_STATE_RST_HANDLING_B, &udev->state_bits)) + return -EBUSY; + + record = &udev->stats.activate_record; + + mutex_lock(&record->lock); + + seq_puts(s, "current time : "); + ubase_dbg_format_time(ktime_get_real_seconds(), s); + seq_puts(s, "\n"); + seq_printf(s, "activate dev count : %llu\n", record->act_cnt); + seq_printf(s, "deactivate dev count : %llu\n", record->deact_cnt); + + total = record->act_cnt + record->deact_cnt; + if (!total) { + seq_puts(s, "activate dev change records : NA\n"); + mutex_unlock(&record->lock); + return 0; + } + + seq_puts(s, "activate dev change records :\n"); + seq_puts(s, "\tNo.\tTIME\t\t\t\tSTATUS\t\tRESULT\n"); + + stats_cnt = min(total, UBASE_ACT_STAT_MAX_NUM); + while (cnt <= stats_cnt) { + total--; + idx = total % UBASE_ACT_STAT_MAX_NUM; + seq_printf(s, "\t%-2d\t", cnt); + ubase_dbg_format_time(record->stats[idx].time, s); + seq_printf(s, "\t%s", record->stats[idx].activate ? + "activate" : "deactivate"); + seq_printf(s, "\t%d", record->stats[idx].result); + seq_puts(s, "\n"); + cnt++; + } + + mutex_unlock(&record->lock); + + return 0; +} + static void ubase_dbg_fill_single_port(struct seq_file *s, struct ubase_perf_stats_result *stats) { @@ -128,6 +399,18 @@ static bool __ubase_dbg_dentry_support(struct device *dev, u32 property) return false; } +/** + * ubase_dbg_dentry_support() - determine whether to create debugfs dentries and debugfs cmd files + * @adev: auxiliary device + * @property: property of debugfs dentry or debufs cmd file + * + * The function is used in the 'support' functions of 'struct ubase_dbg_cmd_info' + * and 'struct ubase_dbg_cmd_info‘ to determine whether to create debugfs dentries + * and debugfs cmd files. + * + * Context: Any context. + * Return: true or false + */ bool ubase_dbg_dentry_support(struct auxiliary_device *adev, u32 property) { if (!adev) @@ -155,6 +438,19 @@ static int __ubase_dbg_seq_file_init(struct device *dev, return 0; } +/** + * ubase_dbg_seq_file_init() - ubase init debugfs cmd file + * @dev: the device + * @dirs: ubase debugfs dentry information + * @dbgfs: ubase debugfs data structure + * @idx: index of dirs + * + * This function is used in the 'init' function within 'struct ubase_dbg_cmd_info' + * to create a ubase debugfs cmd file. + * + * Context: Any context. + * Return: 0 on success, negative error code otherwise + */ int ubase_dbg_seq_file_init(struct device *dev, struct ubase_dbg_dentry_info *dirs, struct ubase_dbgfs *dbgfs, u32 idx) @@ -167,6 +463,11 @@ int ubase_dbg_seq_file_init(struct device *dev, EXPORT_SYMBOL(ubase_dbg_seq_file_init); static struct ubase_dbg_dentry_info ubase_dbg_dentry[] = { + { + .name = "context", + .property = UBASE_SUP_URMA | UBASE_SUP_CDMA | UBASE_SUP_UBL_ETH, + .support = __ubase_dbg_dentry_support, + }, { .name = "qos", .property = UBASE_SUP_URMA | UBASE_SUP_CDMA | UBASE_SUP_UBL_ETH, @@ -193,6 +494,87 @@ static struct ubase_dbg_cmd_info ubase_dbg_cmd[] = { .init = __ubase_dbg_seq_file_init, .read_func = ubase_dbg_dump_rst_info, }, + { + .name = "aeq_context", + .dentry_index = UBASE_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_URMA | UBASE_SUP_CDMA | UBASE_SUP_UBL_ETH, + .support = __ubase_dbg_dentry_support, + .init = __ubase_dbg_seq_file_init, + .read_func = ubase_dbg_dump_aeq_context, + }, + { + .name = "ceq_context", + .dentry_index = UBASE_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_URMA | UBASE_SUP_CDMA | UBASE_SUP_UBL_ETH, + .support = __ubase_dbg_dentry_support, + .init = __ubase_dbg_seq_file_init, + .read_func = ubase_dbg_dump_ceq_context, + }, + { + .name = "caps_info", + .dentry_index = UBASE_DBG_DENTRY_ROOT, + .property = UBASE_SUP_URMA | UBASE_SUP_CDMA | UBASE_SUP_PMU | + UBASE_SUP_UBL_ETH, + .support = __ubase_dbg_dentry_support, + .init = __ubase_dbg_seq_file_init, + .read_func = ubase_dbg_dump_dev_caps, + }, + { + .name = "UBCL_config", + .dentry_index = UBASE_DBG_DENTRY_ROOT, + .property = UBASE_SUP_URMA | UBASE_SUP_CDMA | UBASE_SUP_UBL_ETH, + .support = __ubase_dbg_dentry_support, + .init = __ubase_dbg_seq_file_init, + .read_func = ubase_dbg_dump_ubcl_config, + }, + { + .name = "activate_record", + .dentry_index = UBASE_DBG_DENTRY_ROOT, + .property = UBASE_SUP_URMA | UBASE_SUP_CDMA | UBASE_SUP_UBL_ETH, + .support = __ubase_dbg_dentry_support, + .init = __ubase_dbg_seq_file_init, + .read_func = ubase_dbg_dump_activate_record, + }, + { + .name = "tpg_context", + .dentry_index = UBASE_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_URMA | UBASE_SUP_UBL_ETH, + .support = __ubase_dbg_dentry_support, + .init = __ubase_dbg_seq_file_init, + .read_func = ubase_dbg_dump_tpg_ctx, + }, + { + .name = "tp_context_hw", + .dentry_index = UBASE_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_URMA | UBASE_SUP_UBL_ETH, + .support = __ubase_dbg_dentry_support, + .init = __ubase_dbg_seq_file_init, + .read_func = ubase_dbg_dump_tp_ctx_hw, + }, + { + .name = "tpg_context_hw", + .dentry_index = UBASE_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_URMA | UBASE_SUP_UBL_ETH, + .support = __ubase_dbg_dentry_support, + .init = __ubase_dbg_seq_file_init, + .read_func = ubase_dbg_dump_tpg_ctx_hw, + }, + { + .name = "aeq_context_hw", + .dentry_index = UBASE_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_URMA | UBASE_SUP_CDMA | UBASE_SUP_UBL_ETH, + .support = __ubase_dbg_dentry_support, + .init = __ubase_dbg_seq_file_init, + .read_func = ubase_dbg_dump_aeq_ctx_hw, + }, + { + .name = "ceq_context_hw", + .dentry_index = UBASE_DBG_DENTRY_CONTEXT, + .property = UBASE_SUP_URMA | UBASE_SUP_CDMA | UBASE_SUP_UBL_ETH, + .support = __ubase_dbg_dentry_support, + .init = __ubase_dbg_seq_file_init, + .read_func = ubase_dbg_dump_ceq_ctx_hw, + }, { .name = "sl_vl_map", .dentry_index = UBASE_DBG_DENTRY_QOS, @@ -257,6 +639,14 @@ static struct ubase_dbg_cmd_info ubase_dbg_cmd[] = { .init = __ubase_dbg_seq_file_init, .read_func = ubase_dbg_dump_adev_qos_info, }, + { + .name = "fst_fvt_rqmt_info", + .dentry_index = UBASE_DBG_DENTRY_QOS, + .property = UBASE_SUP_URMA | UBASE_SUP_CDMA | UBASE_SUP_UBL_ETH, + .support = __ubase_dbg_dentry_support, + .init = __ubase_dbg_seq_file_init, + .read_func = ubase_dbg_dump_fsv_fvt_rqmt, + }, { .name = "tm_queue", .dentry_index = UBASE_DBG_DENTRY_QOS, @@ -349,6 +739,18 @@ static int ubase_dbg_create_file(struct device *dev, struct ubase_dbgfs *dbgfs, return 0; } +/** + * ubase_dbg_create_dentry() - ubase debugfs create dentry + * @dev: the device + * @dbgfs: ubase debugfs data structure + * @dirs: ubase debugfs dentry information + * @root_idx: index of the root dentry in dirs, and the root dentry must be the last one in the path + * + * This function is used to create a ubase debugfs cmd file. + * + * Context: Any context. + * Return: 0 on success, negative error code otherwise + */ int ubase_dbg_create_dentry(struct device *dev, struct ubase_dbgfs *dbgfs, struct ubase_dbg_dentry_info *dirs, u32 root_idx) { @@ -429,6 +831,15 @@ void ubase_dbg_unregister_debugfs(void) debugfs_remove_recursive(ubase_dbgfs_root); } +/** + * ubase_diag_debugfs_root() - get ubase debugfs root dentry + * @adev: auxiliary device + * + * This function is used to get ubase debugfs root dentry. + * + * Context: Any context. + * Return: NULL if the adev is empty, otherwise the pointer to struct dentry + */ struct dentry *ubase_diag_debugfs_root(struct auxiliary_device *adev) { if (!adev) @@ -438,6 +849,17 @@ struct dentry *ubase_diag_debugfs_root(struct auxiliary_device *adev) } EXPORT_SYMBOL(ubase_diag_debugfs_root); +/** + * ubase_dbg_format_time() - formatted the time output to seq file + * @time: time value + * @s: seq_file + * + * The function outputs the time in the format of + * 'week month day hour:minute:second year' to seq_file. + * + * Context: Any context. + * Return: 0 on success, negative error code otherwise + */ int ubase_dbg_format_time(time64_t time, struct seq_file *s) { #define YEAR_OFFSET 1900 diff --git a/drivers/ub/ubase/debugfs/ubase_qos_debugfs.c b/drivers/ub/ubase/debugfs/ubase_qos_debugfs.c index 9b8221e422dad55aac05b65d799ccd7c564940a8..91e05df180bb94ac0efc12a1b45f5aedc681aee9 100644 --- a/drivers/ub/ubase/debugfs/ubase_qos_debugfs.c +++ b/drivers/ub/ubase/debugfs/ubase_qos_debugfs.c @@ -252,6 +252,108 @@ int ubase_dbg_dump_adev_qos_info(struct seq_file *s, void *data) return 0; } +static void ubase_dbg_fill_fst_fvt(struct seq_file *s, + struct ubase_query_fst_fvt_rqmt_cmd *resp) +{ + seq_puts(s, "\tFST:\n"); + seq_printf(s, "\t\tsl_queue_vl_num: %u\n", + le16_to_cpu(resp->sl_queue_vl_num)); + seq_printf(s, "\t\tsl_queue_start_qid: %u\n", + le16_to_cpu(resp->sl_queue_start_qid)); + seq_puts(s, "\tFVT:\n"); + seq_printf(s, "\t\tfvt_vl_size: %u\n", le16_to_cpu(resp->fvt_vl_size)); + seq_printf(s, "\t\tfvt_rqmt_offset: %u\n", + le16_to_cpu(resp->fvt_rqmt_offset)); +} + +static void ubase_dbg_fill_fst_revert(struct seq_file *s, + struct ubase_query_fst_fvt_rqmt_cmd *resp) +{ + u16 vl_num = min(UBASE_MAX_VL_NUM, le16_to_cpu(resp->sl_queue_vl_num)); + u16 j; + + seq_puts(s, "\tFST_REVERT:\n"); + seq_puts(s, "\t\tFST_IDX UE_IDX QUE_IDX VL_NUM\n"); + + for (j = 0; j < vl_num; j++) { + seq_puts(s, "\t\t"); + seq_printf(s, "%-9u", le16_to_cpu(resp->fstr_info[j].fst_idx)); + seq_printf(s, "%-10u", resp->fstr_info[j].queue_ue_num); + seq_printf(s, "%-10u", resp->fstr_info[j].queue_que_num); + seq_printf(s, "%-9u", le16_to_cpu(resp->fstr_info[j].queue_vl_num)); + seq_puts(s, "\n"); + } +} + +static void ubase_dbg_fill_rqmt(struct seq_file *s, + struct ubase_query_fst_fvt_rqmt_cmd *resp) +{ + u16 vl_size = min(UBASE_MAX_VL_NUM, le16_to_cpu(resp->fvt_vl_size)); + u16 j; + + seq_puts(s, "\tRQMT:\n"); + seq_puts(s, "\t\tFST_IDX QUE_IDX QUE_SHIFT\n"); + + for (j = 0; j < vl_size; j++) { + seq_puts(s, "\t\t"); + seq_printf(s, "%-9u", le16_to_cpu(resp->rqmt_info[j].fst_idx)); + seq_printf(s, "%-10u", + le16_to_cpu(resp->rqmt_info[j].start_queue_idx)); + seq_printf(s, "%-12u", + le16_to_cpu(resp->rqmt_info[j].queue_quantity_shift)); + seq_puts(s, "\n"); + } + + seq_puts(s, "\n"); +} + +static void ubase_dbg_fill_tbl_content(struct seq_file *s, + struct ubase_query_fst_fvt_rqmt_cmd *resp) +{ + ubase_dbg_fill_fst_fvt(s, resp); + + ubase_dbg_fill_fst_revert(s, resp); + + ubase_dbg_fill_rqmt(s, resp); +} + +int ubase_dbg_dump_fsv_fvt_rqmt(struct seq_file *s, void *data) +{ + struct ubase_dev *udev = dev_get_drvdata(s->private); + struct ubase_query_fst_fvt_rqmt_cmd resp = {0}; + struct ubase_ue_node *ue_node; + u16 ue_id; + int ret; + + if (!test_bit(UBASE_STATE_INITED_B, &udev->state_bits) || + test_bit(UBASE_STATE_RST_HANDLING_B, &udev->state_bits)) + return -EBUSY; + + seq_puts(s, "current ue:\n"); + ret = ubase_query_fst_fvt_rqmt(udev, &resp, 0); + if (ret) + return ret; + + ubase_dbg_fill_tbl_content(s, &resp); + + mutex_lock(&udev->ue_list_lock); + list_for_each_entry(ue_node, &udev->ue_list, list) { + ue_id = ue_node->bus_ue_id; + memset(&resp, 0, sizeof(resp)); + + seq_printf(s, "ue%u:\n", ue_id); + + ret = ubase_query_fst_fvt_rqmt(udev, &resp, ue_id); + if (ret) + goto out; + ubase_dbg_fill_tbl_content(s, &resp); + } + +out: + mutex_unlock(&udev->ue_list_lock); + return ret; +} + static void ubase_dbg_fill_tm_queue_seq(struct seq_file *s, struct ubase_query_tm_queue_cmd *resp) { diff --git a/drivers/ub/ubase/debugfs/ubase_qos_debugfs.h b/drivers/ub/ubase/debugfs/ubase_qos_debugfs.h index 48f45a2dbec065b58a49e03417c509e1431ea3f9..e44b4cacd21e1b589bd0b4639fb65443dd085ce6 100644 --- a/drivers/ub/ubase/debugfs/ubase_qos_debugfs.h +++ b/drivers/ub/ubase/debugfs/ubase_qos_debugfs.h @@ -16,6 +16,7 @@ int ubase_dbg_dump_ets_tcg_info(struct seq_file *s, void *data); int ubase_dbg_dump_ets_port_info(struct seq_file *s, void *data); int ubase_dbg_dump_rack_vl_bitmap(struct seq_file *s, void *data); int ubase_dbg_dump_adev_qos_info(struct seq_file *s, void *data); +int ubase_dbg_dump_fsv_fvt_rqmt(struct seq_file *s, void *data); int ubase_dbg_dump_tm_queue_info(struct seq_file *s, void *data); int ubase_dbg_dump_tm_qset_info(struct seq_file *s, void *data); int ubase_dbg_dump_tm_pri_info(struct seq_file *s, void *data); diff --git a/drivers/ub/ubase/ubase_cmd.c b/drivers/ub/ubase/ubase_cmd.c index 4526f92ac1177d5e24409ae475a52395e7be5fea..ab554343136f84b47e9c53ec23261425b3d7018e 100644 --- a/drivers/ub/ubase/ubase_cmd.c +++ b/drivers/ub/ubase/ubase_cmd.c @@ -306,10 +306,11 @@ int ubase_send_cmd(struct ubase_dev *udev, return ret; } -static int ubase_cmd_query_version(struct ubase_dev *udev, u32 *fw_version) +static int ubase_cmd_query_version(struct ubase_dev *udev) { struct ubase_query_version_cmd *resp; struct ubase_cmdq_desc desc; + u32 fw_ver; int ret; ubase_cmd_setup_basic_desc(&desc, UBASE_OPC_QUERY_FW_VER, true, 1); @@ -321,13 +322,14 @@ static int ubase_cmd_query_version(struct ubase_dev *udev, u32 *fw_version) } resp = (struct ubase_query_version_cmd *)desc.data; - *fw_version = le32_to_cpu(resp->firmware); + udev->caps.dev_caps.fw_version = le32_to_cpu(resp->fw_version); + fw_ver = udev->caps.dev_caps.fw_version; ubase_info(udev, "The firmware version is %u.%u.%u.%u\n", - u32_get_bits(*fw_version, UBASE_FW_VERSION_BYTE3_MASK), - u32_get_bits(*fw_version, UBASE_FW_VERSION_BYTE2_MASK), - u32_get_bits(*fw_version, UBASE_FW_VERSION_BYTE1_MASK), - u32_get_bits(*fw_version, UBASE_FW_VERSION_BYTE0_MASK)); + u32_get_bits(fw_ver, UBASE_FW_VERSION_BYTE3_MASK), + u32_get_bits(fw_ver, UBASE_FW_VERSION_BYTE2_MASK), + u32_get_bits(fw_ver, UBASE_FW_VERSION_BYTE1_MASK), + u32_get_bits(fw_ver, UBASE_FW_VERSION_BYTE0_MASK)); return 0; } @@ -366,7 +368,7 @@ int ubase_cmd_init(struct ubase_dev *udev) clear_bit(UBASE_STATE_CMD_DISABLE, &udev->hw.state); - ret = ubase_cmd_query_version(udev, &udev->caps.dev_caps.fw_version); + ret = ubase_cmd_query_version(udev); if (ret) goto err_query_version; @@ -775,9 +777,21 @@ int __ubase_cmd_send_in(struct ubase_dev *udev, struct ubase_cmd_buf *in) } /** - * When uninstalling, cmdq needs to be successfully sended as much as possible, - * but the cmd may be disabled during reset, this interface attempts to send cmd - * when it is enabled. + * ubase_cmd_send_inout_ex() - query(and write) cmd extension function + * @aux_dev: auxiliary device + * @in: the intput cmd buff + * @out: the output cmd buff + * @time_out: timeout duration, unit: ms + * + * When the timeout parameter is set to 0, this function behaves the same as + * 'ubase_cmd_send_inout'. When the timeout parameter is not 0, if the cmdq + * channel is disabled and it recovers within the timeout period, the cmdq can + * still process commands normally. + * This function is applicable to scenarios such as concurrent resets, where the + * cmdq channel is first set to be disabled and then restored to normal operation. + * + * Context: Process context. Takes and releases , BH-safe. May sleep. + * Return: 0 on success, negative error code otherwise */ int ubase_cmd_send_inout_ex(struct auxiliary_device *aux_dev, struct ubase_cmd_buf *in, struct ubase_cmd_buf *out, @@ -810,6 +824,22 @@ int ubase_cmd_send_inout_ex(struct auxiliary_device *aux_dev, } EXPORT_SYMBOL(ubase_cmd_send_inout_ex); +/** + * ubase_cmd_send_inout() - query(and write) cmd function + * @aux_dev: auxiliary device + * @in: the intput cmd buff + * @out: the output cmd buff + * + * The firmware determines the processing behavior based on 'in->opcode'. + * 'in->data_size' and 'in->data' represent the length of valid data and the + * address where the data is stored for interaction with the firmware. + * 'in->is_read' determines whether to read the query results. + * 'out->data_size' and 'out->data' represent the length of valid data and the + * address where the data is stored for the reading the query results. + * + * Context: Process context. Takes and releases , BH-safe. + * Return: 0 on success, negative error code otherwise + */ int ubase_cmd_send_inout(struct auxiliary_device *aux_dev, struct ubase_cmd_buf *in, struct ubase_cmd_buf *out) @@ -821,6 +851,22 @@ int ubase_cmd_send_inout(struct auxiliary_device *aux_dev, } EXPORT_SYMBOL(ubase_cmd_send_inout); +/** + * ubase_cmd_send_in_ex() - write cmd extension function + * @aux_dev: auxiliary device + * @in: the intput cmd buff + * @time_out: timeout duration, unit: ms + * + * When the timeout parameter is set to 0, this function behaves the same as + * 'ubase_cmd_send_in'. When the timeout parameter is not 0, if the cmdq + * channel is disabled and it recovers within the timeout period, the cmdq can + * still process commands normally. + * This function is applicable to scenarios such as concurrent resets, where the + * cmdq channel is first set to disabled and then restored to normal operation. + * + * Context: Process context. Takes and releases , BH-safe. May sleep. + * Return: 0 on success, negative error code otherwise + */ int ubase_cmd_send_in_ex(struct auxiliary_device *aux_dev, struct ubase_cmd_buf *in, u32 time_out) { @@ -835,6 +881,19 @@ int ubase_cmd_send_in_ex(struct auxiliary_device *aux_dev, } EXPORT_SYMBOL(ubase_cmd_send_in_ex); +/** + * ubase_cmd_send_in() - write cmd function + * @aux_dev: auxiliary device + * @in: the intput cmd buff + * + * This function is only used for writing cmdq opcodes. 'in->is_read' must be + * false. The firmware determines the processing behavior based on 'in->opcode'. + * 'in->data_size' and 'in->data' represent the length of valid data and the + * address where the data is stored for interaction with the firmware. + * + * Context: Process context. Takes and releases , BH-safe. + * Return: 0 on success, negative error code otherwise + */ int ubase_cmd_send_in(struct auxiliary_device *aux_dev, struct ubase_cmd_buf *in) { @@ -876,6 +935,19 @@ static int __ubase_cmd_get_data_size(struct ubase_dev *udev, u16 opcode, return 0; } +/** + * ubase_cmd_get_data_size() - obtain the valid data length from cmdq opcode + * @aux_dev: auxiliary device + * @opcode: cmdq opcode + * @data_size: Save the valid data length of cmdq opcode + * + * For each opcode, the firmware has a corresponding valid data length. + * This function queries the firmware to obtain the valid data length + * corresponding to the opcode. + * + * Context: Process context. Takes and releases , BH-safe. + * Return: 0 on success, negative error code otherwise + */ int ubase_cmd_get_data_size(struct auxiliary_device *aux_dev, u16 opcode, u16 *data_size) { @@ -923,6 +995,18 @@ int __ubase_register_crq_event(struct ubase_dev *udev, return ret; } +/** + * ubase_register_crq_event() - register crq event processing function + * @aux_dev: auxiliary device + * @nb: the crq event notification block + * + * Register the crq handler function. When the firmware reports a crq event, + * if the opcode reported by the firmware matches the registered 'nb->opcode', + * the 'nb->crq_handler' function will be called to process it. + * + * Context: Any context. + * Return: 0 on success, negative error code otherwise + */ int ubase_register_crq_event(struct auxiliary_device *aux_dev, struct ubase_crq_event_nb *nb) { @@ -953,6 +1037,16 @@ void __ubase_unregister_crq_event(struct ubase_dev *udev, u16 opcode) mutex_unlock(&crq_table->lock); } +/** + * ubase_unregister_crq_event() - unregister crq event processing function + * @aux_dev: auxiliary device + * @opcode: cmdq crq opcode + * + * Unregisters the crq processing function. This function is called when user + * no longer want to handle the crq opcode events reported by the firmware. + * + * Context: Any context. + */ void ubase_unregister_crq_event(struct auxiliary_device *aux_dev, u16 opcode) { struct ubase_dev *udev; diff --git a/drivers/ub/ubase/ubase_cmd.h b/drivers/ub/ubase/ubase_cmd.h index 0187597493b7910dabe742265b1c6f6fd98bee02..a99422b59dc4b44863e4432dadbb60666338a0a3 100644 --- a/drivers/ub/ubase/ubase_cmd.h +++ b/drivers/ub/ubase/ubase_cmd.h @@ -43,10 +43,26 @@ enum ubase_cmd_state { }; struct ubase_query_version_cmd { - __le32 firmware; - __le32 hardware; - __le32 rsv; - __le32 caps[UBASE_CAP_LEN]; + __le32 fw_version; + u8 rsv[20]; +}; + +enum ubase_drv_cap_bit { + UBASE_CAP_SUP_ACTIVATE_B = 0, +}; + +struct ubase_notify_drv_cap_cmd { + u8 cap_bits[24]; /* see ubase_drv_cap_bit */ +}; + +#define UBASE_UBCL_CFG_DATA_ALIGN 4 +#define UBASE_UBCL_CFG_DATA_NUM 60 +struct ubase_ubcl_config_cmd { + __le16 is_query_size; + __le16 offset; + __le16 size; + __le16 rsv; + __le32 data[UBASE_UBCL_CFG_DATA_NUM]; }; enum ubase_ue2ue_sub_cmd { @@ -67,7 +83,8 @@ struct ubase_ue2ue_ctrlq_head { u16 out_size; u8 need_resp : 1; u8 is_resp : 1; - u8 rsv : 6; + u8 is_async : 1; + u8 rsv : 5; }; struct ubase_start_perf_stats_cmd { @@ -96,10 +113,10 @@ struct ubase_cfg_ets_vl_sch_cmd { }; struct ubase_cfg_tm_vl_sch_cmd { - __le16 bus_ue_id; + u8 rsvd0[2]; __le16 vl_bitmap; __le16 vl_tsa; - u8 rsvd[2]; + u8 rsvd1[2]; u8 vl_bw[UBASE_MAX_VL_NUM]; }; diff --git a/drivers/ub/ubase/ubase_ctrlq.c b/drivers/ub/ubase/ubase_ctrlq.c index d590463b0efca062533a2f492c0aaa4682ab25b7..5dcb25012d617db5ea4cb46dc315ab372932b47f 100644 --- a/drivers/ub/ubase/ubase_ctrlq.c +++ b/drivers/ub/ubase/ubase_ctrlq.c @@ -11,19 +11,103 @@ #include "ubase_cmd.h" #include "ubase_ctrlq.h" -static inline void ubase_ctrlq_crq_table_init(struct ubase_dev *udev) +/* UNIC ctrlq msg white list */ +static const struct ubase_ctrlq_event_nb ubase_ctrlq_wlist_unic[] = { + {UBASE_CTRLQ_SER_TYPE_IP_ACL, UBASE_CTRLQ_OPC_NOTIFY_IP, NULL, NULL}, +}; + +/* UDMA ctrlq msg white list */ +static const struct ubase_ctrlq_event_nb ubase_ctrlq_wlist_udma[] = { + {UBASE_CTRLQ_SER_TYPE_TP_ACL, UBASE_CTRLQ_OPC_CHECK_TP_ACTIVE, NULL, NULL}, + {UBASE_CTRLQ_SER_TYPE_DEV_REGISTER, UBASE_CTRLQ_OPC_UPDATE_SEID, NULL, NULL}, + {UBASE_CTRLQ_SER_TYPE_DEV_REGISTER, UBASE_CTRLQ_OPC_UPDATE_UE_SEID_GUID, NULL, NULL}, + {UBASE_CTRLQ_SER_TYPE_DEV_REGISTER, UBASE_CTRLQ_OPC_NOTIFY_RES_RATIO, NULL, NULL}, +}; + +/* CDMA ctrlq msg white list */ +static const struct ubase_ctrlq_event_nb ubase_ctrlq_wlist_cdma[] = { + {UBASE_CTRLQ_SER_TYPE_DEV_REGISTER, UBASE_CTRLQ_OPC_UPDATE_SEID, NULL, NULL}, +}; + +static int ubase_ctrlq_alloc_crq_tbl_mem(struct ubase_dev *udev) { struct ubase_ctrlq_crq_table *crq_tab = &udev->ctrlq.crq_table; + u16 cnt = 0; + + if (ubase_dev_cdma_supported(udev)) { + cnt = ARRAY_SIZE(ubase_ctrlq_wlist_cdma); + } else if (ubase_dev_urma_supported(udev)) { + if (ubase_dev_unic_supported(udev)) + cnt += ARRAY_SIZE(ubase_ctrlq_wlist_unic); + if (ubase_dev_udma_supported(udev)) + cnt += ARRAY_SIZE(ubase_ctrlq_wlist_udma); + } + + if (!cnt) + return -EINVAL; + + crq_tab->crq_nbs = kcalloc(cnt, sizeof(struct ubase_ctrlq_event_nb), GFP_KERNEL); + if (!crq_tab->crq_nbs) + return -ENOMEM; + + crq_tab->crq_nb_cnt = cnt; + + return 0; +} + +static void ubase_ctrlq_free_crq_tbl_mem(struct ubase_dev *udev) +{ + struct ubase_ctrlq_crq_table *crq_tab = &udev->ctrlq.crq_table; + + kfree(crq_tab->crq_nbs); + crq_tab->crq_nbs = NULL; + crq_tab->crq_nb_cnt = 0; +} + +static void ubase_ctrlq_init_crq_wlist(struct ubase_dev *udev) +{ + struct ubase_ctrlq_crq_table *crq_tab = &udev->ctrlq.crq_table; + u32 offset = 0; + + if (ubase_dev_cdma_supported(udev)) { + memcpy(crq_tab->crq_nbs, ubase_ctrlq_wlist_cdma, + sizeof(ubase_ctrlq_wlist_cdma)); + } else if (ubase_dev_urma_supported(udev)) { + if (ubase_dev_unic_supported(udev)) { + memcpy(crq_tab->crq_nbs, ubase_ctrlq_wlist_unic, + sizeof(ubase_ctrlq_wlist_unic)); + offset = ARRAY_SIZE(ubase_ctrlq_wlist_unic); + } + if (ubase_dev_udma_supported(udev)) { + memcpy(&crq_tab->crq_nbs[offset], ubase_ctrlq_wlist_udma, + sizeof(ubase_ctrlq_wlist_udma)); + } + } +} + +static int ubase_ctrlq_crq_table_init(struct ubase_dev *udev) +{ + struct ubase_ctrlq_crq_table *crq_tab = &udev->ctrlq.crq_table; + int ret; + + ret = ubase_ctrlq_alloc_crq_tbl_mem(udev); + if (ret) + return ret; + + ubase_ctrlq_init_crq_wlist(udev); mutex_init(&crq_tab->lock); - INIT_LIST_HEAD(&crq_tab->crq_nbs.list); + + return 0; } -static inline void ubase_ctrlq_crq_table_uninit(struct ubase_dev *udev) +static void ubase_ctrlq_crq_table_uninit(struct ubase_dev *udev) { struct ubase_ctrlq_crq_table *crq_tab = &udev->ctrlq.crq_table; mutex_destroy(&crq_tab->lock); + + ubase_ctrlq_free_crq_tbl_mem(udev); } static inline u16 ubase_ctrlq_msg_queue_depth(struct ubase_dev *udev) @@ -233,10 +317,15 @@ int ubase_ctrlq_init(struct ubase_dev *udev) if (ret) goto err_msg_queue_init; + ret = ubase_ctrlq_crq_table_init(udev); + if (ret) + goto err_crq_table_init; + udev->ctrlq.csq_next_seq = 1; atomic_set(&udev->ctrlq.req_cnt, 0); - ubase_ctrlq_crq_table_init(udev); +err_crq_table_init: + ubase_ctrlq_msg_queue_uninit(udev); success: set_bit(UBASE_CTRLQ_STATE_ENABLE, &udev->ctrlq.state); return 0; @@ -369,11 +458,13 @@ static void ubase_ctrlq_fill_first_bb(struct ubase_dev *udev, head->service_type = msg->service_type; head->opcode = msg->opcode; head->mbx_ue_id = ue_info ? ue_info->mbx_ue_id : 0; - head->ret = msg->is_resp ? msg->resp_ret : 0; + head->ret = ubase_ctrlq_msg_is_resp(msg) ? msg->resp_ret : 0; head->bus_ue_id = cpu_to_le16(ue_info ? ue_info->bus_ue_id : ue->entity_idx); - memcpy(head->data, msg->in, min(msg->in_size, UBASE_CTRLQ_DATA_LEN)); + if (msg->in) + memcpy(head->data, msg->in, + min(msg->in_size, UBASE_CTRLQ_DATA_LEN)); } static inline void ubase_ctrlq_csq_report_irq(struct ubase_dev *udev) @@ -408,10 +499,12 @@ static int ubase_ctrlq_send_to_cmdq(struct ubase_dev *udev, ue2ue_head.out_size = msg->out_size; ue2ue_head.need_resp = msg->need_resp; ue2ue_head.is_resp = msg->is_resp; + ue2ue_head.is_async = msg->is_async; memcpy(req, &ue2ue_head, sizeof(ue2ue_head)); memcpy((u8 *)req + sizeof(ue2ue_head), head, UBASE_CTRLQ_HDR_LEN); - memcpy((u8 *)req + sizeof(ue2ue_head) + UBASE_CTRLQ_HDR_LEN, msg->in, - msg->in_size); + if (msg->in) + memcpy((u8 *)req + sizeof(ue2ue_head) + UBASE_CTRLQ_HDR_LEN, + msg->in, msg->in_size); __ubase_fill_inout_buf(&in, UBASE_OPC_UE2UE_UBASE, false, req_len, req); ret = __ubase_cmd_send_in(udev, &in); @@ -541,18 +634,17 @@ static void ubase_ctrlq_addto_msg_queue(struct ubase_dev *udev, u16 seq, { struct ubase_ctrlq_msg_ctx *ctx; - if (!msg->need_resp) + if (!(ubase_ctrlq_msg_is_sync_req(msg) || + ubase_ctrlq_msg_is_async_req(msg))) return; ctx = &udev->ctrlq.msg_queue[seq]; ctx->valid = 1; - ctx->is_sync = msg->need_resp && msg->out && msg->out_size ? 1 : 0; + ctx->is_sync = ubase_ctrlq_msg_is_sync_req(msg) ? 1 : 0; ctx->result = ETIME; ctx->dead_jiffies = jiffies + msecs_to_jiffies(UBASE_CTRLQ_DEAD_TIME); - if (ctx->is_sync) { - ctx->out = msg->out; - ctx->out_size = msg->out_size; - } + ctx->out = msg->out; + ctx->out_size = msg->out_size; if (ue_info) { ctx->ue_seq = ue_info->seq; @@ -564,25 +656,58 @@ static void ubase_ctrlq_addto_msg_queue(struct ubase_dev *udev, u16 seq, static int ubase_ctrlq_msg_check(struct ubase_dev *udev, struct ubase_ctrlq_msg *msg) { - if (!msg || !msg->in || !msg->in_size) { - ubase_err(udev, "ctrlq input buf is invalid.\n"); + if ((!msg->in && msg->in_size) || (msg->in && !msg->in_size)) { + ubase_err(udev, "ctrlq msg in param error.\n"); return -EINVAL; } - if (msg->is_resp && !(msg->resp_seq & UBASE_CTRLQ_SEQ_MASK)) { - ubase_err(udev, "ctrlq input resp_seq(%u) is invalid.\n", - msg->resp_seq); + if ((!msg->out && msg->out_size) || (msg->out && !msg->out_size)) { + ubase_err(udev, "ctrlq msg out param error.\n"); return -EINVAL; } if (msg->in_size > UBASE_CTRLQ_MAX_DATA_SIZE) { ubase_err(udev, - "requested ctrlq space(%u) exceeds the maximum(%u).\n", + "ctrlq msg in_size(%u) exceeds the maximum(%u).\n", msg->in_size, UBASE_CTRLQ_MAX_DATA_SIZE); return -EINVAL; } - return 0; + if (ubase_ctrlq_msg_is_sync_req(msg)) + return 0; + + if (ubase_ctrlq_msg_is_async_req(msg)) { + if (msg->out) { + ubase_err(udev, "ctrlq msg out is not NULL in async req.\n"); + return -EINVAL; + } + return 0; + } + + if (ubase_ctrlq_msg_is_notify_req(msg)) { + if (msg->out) { + ubase_err(udev, "ctrlq msg out is not NULL in notify req.\n"); + return -EINVAL; + } + return 0; + } + + if (ubase_ctrlq_msg_is_resp(msg)) { + if (msg->out) { + ubase_err(udev, "ctrlq msg out is not NULL in resp.\n"); + return -EINVAL; + } + if (!(msg->resp_seq & UBASE_CTRLQ_SEQ_MASK)) { + ubase_err(udev, "ctrlq msg resp_seq error, resp_seq=%u.\n", + msg->resp_seq); + return -EINVAL; + } + return 0; + } + + ubase_err(udev, "ctrlq msg param error, is_resp=%u, is_async=%u, need_resp=%u.\n", + msg->is_resp, msg->is_async, msg->need_resp); + return -EINVAL; } static int ubase_ctrlq_check_csq_enough(struct ubase_dev *udev, u16 num) @@ -607,8 +732,6 @@ static int ubase_ctrlq_send_real(struct ubase_dev *udev, struct ubase_ctrlq_msg *msg, struct ubase_ctrlq_ue_info *ue_info) { - bool sync_req = msg->out && msg->out_size && msg->need_resp; - bool no_resp = !msg->is_resp && !msg->need_resp; struct ubase_ctrlq_ring *csq = &udev->ctrlq.csq; struct ubase_ctrlq_base_block head = {0}; u16 seq, num; @@ -622,7 +745,7 @@ static int ubase_ctrlq_send_real(struct ubase_dev *udev, if (ret) goto unlock; - if (!msg->is_resp) { + if (!ubase_ctrlq_msg_is_resp(msg)) { ret = ubase_ctrlq_alloc_seq(udev, msg, &seq); if (ret) { ubase_warn(udev, "no enough seq in ctrlq.\n"); @@ -640,20 +763,22 @@ static int ubase_ctrlq_send_real(struct ubase_dev *udev, ret = ubase_ctrlq_send_msg_to_sq(udev, &head, msg, num); if (ret) { spin_unlock_bh(&csq->lock); - goto free_seq; + if (!ubase_ctrlq_msg_is_resp(msg)) + ubase_ctrlq_free_seq(udev, seq); + return ret; } spin_unlock_bh(&csq->lock); - if (sync_req) + if (ubase_ctrlq_msg_is_sync_req(msg)) ret = ubase_ctrlq_wait_completed(udev, seq, msg); -free_seq: - /* Only the seqs in synchronous requests and no response requests need to be released. */ - /* The seqs are released in periodic tasks of asynchronous requests. */ - if (sync_req || no_resp) + if (ubase_ctrlq_msg_is_sync_req(msg) || + ubase_ctrlq_msg_is_notify_req(msg)) ubase_ctrlq_free_seq(udev, seq); + return ret; + unlock: spin_unlock_bh(&csq->lock); return ret; @@ -700,6 +825,28 @@ int __ubase_ctrlq_send(struct ubase_dev *udev, struct ubase_ctrlq_msg *msg, return ret; } +/** + * ubase_ctrlq_send_msg() - ctrlq message send function + * @aux_dev: auxiliary device + * @msg: the message to be sent + * + * The driver uses this function to send a ctrlq message to the management software. + * The management software determines the module responsible for processing the message + * based on 'msg->service_ver', 'msg->service_type', and 'msg->opcode'; + * it also retrieves the length and content of the data to be sent from + * 'msg->in_size' and 'msg->in'. + * When 'msg->is_resp' is set to 1, it indicates that the message is a response + * to a ctrlq message from the management software. 'msg->resp_seq' and 'msg->resp_ret' + * represent the sequence number and processing result of the ctrlq message from + * the management software. + * When 'msg->need_resp' is set to 1, it indicates that the management software needs + * to respond to the driver's ctrlq message. If 'msg->out_size' is not zero and + * 'msg->out' is not empty, this function will wait synchronously for the management + * software's response, and the response information will be stored in 'msg->out'. + * + * Context: Process context. Takes and releases , BH-safe. May sleep. + * Return: 0 on success, negative error code otherwise + */ int ubase_ctrlq_send_msg(struct auxiliary_device *aux_dev, struct ubase_ctrlq_msg *msg) { @@ -742,27 +889,68 @@ static void ubase_ctrlq_read_msg_data(struct ubase_dev *udev, u8 num, u8 *msg) } } +static void ubase_ctrlq_send_unsupported_resp(struct ubase_dev *udev, + struct ubase_ctrlq_base_block *head, + u16 resp_seq, u8 resp_ret) +{ + struct ubase_ctrlq_msg msg = {0}; + int ret; + + msg.service_ver = head->service_ver; + msg.service_type = head->service_type; + msg.opcode = head->opcode; + msg.is_resp = 1; + msg.resp_seq = resp_seq; + msg.resp_ret = resp_ret; + + ret = __ubase_ctrlq_send(udev, &msg, NULL); + if (ret) + ubase_warn(udev, "failed to send ctrlq unsupport resp, ret=%d.", + ret); +} + static void ubase_ctrlq_crq_event_callback(struct ubase_dev *udev, struct ubase_ctrlq_base_block *head, void *msg_data, u16 msg_data_len, u16 seq) { +#define EDRVNOEXIST 255 struct ubase_ctrlq_crq_table *crq_tab = &udev->ctrlq.crq_table; - struct ubase_ctrlq_crq_event_nbs *nbs; + int ret = -ENOENT; + u32 i; + + ubase_info(udev, + "ctrlq recv notice req: seq=%u, ser_type=%u, ser_ver=%u, opc=0x%x.", + seq, head->service_type, head->service_ver, head->opcode); mutex_lock(&crq_tab->lock); - list_for_each_entry(nbs, &crq_tab->crq_nbs.list, list) { - if (nbs->crq_nb.service_type == head->service_type && - nbs->crq_nb.opcode == head->opcode) { - nbs->crq_nb.crq_handler(nbs->crq_nb.back, - head->service_ver, - msg_data, - msg_data_len, - seq); + for (i = 0; i < crq_tab->crq_nb_cnt; i++) { + if (crq_tab->crq_nbs[i].service_type == head->service_type && + crq_tab->crq_nbs[i].opcode == head->opcode) { + if (!crq_tab->crq_nbs[i].crq_handler) { + ret = -EDRVNOEXIST; + break; + } + ret = crq_tab->crq_nbs[i].crq_handler(crq_tab->crq_nbs[i].back, + head->service_ver, + msg_data, + msg_data_len, + seq); break; } } mutex_unlock(&crq_tab->lock); + + if (ret == -ENOENT) { + ubase_info(udev, "this notice is not supported."); + ubase_ctrlq_send_unsupported_resp(udev, head, seq, EOPNOTSUPP); + } else if (ret == -EOPNOTSUPP) { + ubase_info(udev, "the notice processor return not support."); + ubase_ctrlq_send_unsupported_resp(udev, head, seq, EOPNOTSUPP); + } else if (ret == -EDRVNOEXIST) { + ubase_info(udev, "the notice processor is unregistered."); + ubase_ctrlq_send_unsupported_resp(udev, head, seq, EDRVNOEXIST); + } } static void ubase_ctrlq_notify_completed(struct ubase_dev *udev, @@ -773,12 +961,13 @@ static void ubase_ctrlq_notify_completed(struct ubase_dev *udev, ctx = &udev->ctrlq.msg_queue[seq]; ctx->result = head->ret; - memcpy(ctx->out, msg, min(msg_len, ctx->out_size)); + if (ctx->out) + memcpy(ctx->out, msg, min(msg_len, ctx->out_size)); complete(&ctx->done); } -static bool ubase_ctrlq_check_seq(struct ubase_dev *udev, u16 seq) +bool ubase_ctrlq_check_seq(struct ubase_dev *udev, u16 seq) { bool is_pushed = !!(seq & UBASE_CTRLQ_SEQ_MASK); u16 max_seq = ubase_ctrlq_max_seq(udev); @@ -993,13 +1182,26 @@ void ubase_ctrlq_clean_service_task(struct ubase_delay_work *ubase_work) spin_unlock_bh(&csq->lock); } + +/** + * ubase_ctrlq_register_crq_event() - register ctrlq crq event processing function + * @aux_dev: auxiliary device + * @nb: the ctrlq crq event notification block + * + * Register the ctrlq crq handler function. When the management software reports + * a ctrlq crq event, if the registered 'nb->opcode' and 'nb->service_type' match + * the crq, the 'nb->crq_handler' function will be called to process it. + * + * Context: Any context. + * Return: 0 on success, negative error code otherwise + */ int ubase_ctrlq_register_crq_event(struct auxiliary_device *aux_dev, struct ubase_ctrlq_event_nb *nb) { - struct ubase_ctrlq_crq_event_nbs *nbs, *tmp, *new_nbs; struct ubase_ctrlq_crq_table *crq_tab; struct ubase_dev *udev; - int ret; + int ret = -ENOENT; + u32 i; if (!aux_dev || !nb || !nb->crq_handler) return -EINVAL; @@ -1007,42 +1209,43 @@ int ubase_ctrlq_register_crq_event(struct auxiliary_device *aux_dev, udev = __ubase_get_udev_by_adev(aux_dev); crq_tab = &udev->ctrlq.crq_table; mutex_lock(&crq_tab->lock); - list_for_each_entry_safe(nbs, tmp, &crq_tab->crq_nbs.list, list) { - if (nbs->crq_nb.service_type == nb->service_type && - nbs->crq_nb.opcode == nb->opcode) { - ret = -EEXIST; - goto err_crq_register; + for (i = 0; i < crq_tab->crq_nb_cnt; i++) { + if (crq_tab->crq_nbs[i].service_type == nb->service_type && + crq_tab->crq_nbs[i].opcode == nb->opcode) { + if (crq_tab->crq_nbs[i].crq_handler) { + ret = -EEXIST; + break; + } + crq_tab->crq_nbs[i].back = nb->back; + crq_tab->crq_nbs[i].crq_handler = nb->crq_handler; + ret = 0; + break; } } - new_nbs = kzalloc(sizeof(*new_nbs), GFP_KERNEL); - if (!new_nbs) { - ret = -ENOMEM; - goto err_crq_register; - } - - new_nbs->crq_nb = *nb; - list_add_tail(&new_nbs->list, &crq_tab->crq_nbs.list); - mutex_unlock(&crq_tab->lock); - - return 0; - -err_crq_register: mutex_unlock(&crq_tab->lock); - ubase_err(udev, - "failed to register ctrlq crq event, opcode = 0x%x, service_type = 0x%x, ret = %d.\n", - nb->opcode, nb->service_type, ret); return ret; } EXPORT_SYMBOL(ubase_ctrlq_register_crq_event); +/** + * ubase_ctrlq_unregister_crq_event() - unregister ctrlq crq event processing function + * @aux_dev: auxiliary device + * @service_type: the ctrlq service type + * @opcode: the ctrlq opcode + * + * Unregisters the ctrlq crq processing function. This function is called when user + * no longer wants to handle the 'service_type' and 'opcode' ctrlq crq events. + * + * Context: Any context. + */ void ubase_ctrlq_unregister_crq_event(struct auxiliary_device *aux_dev, u8 service_type, u8 opcode) { - struct ubase_ctrlq_crq_event_nbs *nbs, *tmp; struct ubase_ctrlq_crq_table *crq_tab; struct ubase_dev *udev; + u32 i; if (!aux_dev) return; @@ -1050,11 +1253,11 @@ void ubase_ctrlq_unregister_crq_event(struct auxiliary_device *aux_dev, udev = __ubase_get_udev_by_adev(aux_dev); crq_tab = &udev->ctrlq.crq_table; mutex_lock(&crq_tab->lock); - list_for_each_entry_safe(nbs, tmp, &crq_tab->crq_nbs.list, list) { - if (nbs->crq_nb.service_type == service_type && - nbs->crq_nb.opcode == opcode) { - list_del(&nbs->list); - kfree(nbs); + for (i = 0; i < crq_tab->crq_nb_cnt; i++) { + if (crq_tab->crq_nbs[i].service_type == service_type && + crq_tab->crq_nbs[i].opcode == opcode) { + crq_tab->crq_nbs[i].back = NULL; + crq_tab->crq_nbs[i].crq_handler = NULL; break; } } diff --git a/drivers/ub/ubase/ubase_ctrlq.h b/drivers/ub/ubase/ubase_ctrlq.h index 431253df50540a95a1eba740c6e70db816e5fbfa..c3d5f55db87ddfbee5ff42bc61f2524a4fc2823a 100644 --- a/drivers/ub/ubase/ubase_ctrlq.h +++ b/drivers/ub/ubase/ubase_ctrlq.h @@ -87,6 +87,26 @@ struct ubase_ctrlq_reset_ctrl_req { u8 rsv[3]; }; +static inline bool ubase_ctrlq_msg_is_sync_req(struct ubase_ctrlq_msg *msg) +{ + return !msg->is_resp && !msg->is_async && msg->need_resp; +} + +static inline bool ubase_ctrlq_msg_is_async_req(struct ubase_ctrlq_msg *msg) +{ + return !msg->is_resp && msg->is_async && msg->need_resp; +} + +static inline bool ubase_ctrlq_msg_is_notify_req(struct ubase_ctrlq_msg *msg) +{ + return !msg->is_resp && !msg->is_async && !msg->need_resp; +} + +static inline bool ubase_ctrlq_msg_is_resp(struct ubase_ctrlq_msg *msg) +{ + return msg->is_resp && !msg->is_async && !msg->need_resp; +} + int ubase_ctrlq_init(struct ubase_dev *udev); void ubase_ctrlq_uninit(struct ubase_dev *udev); void ubase_ctrlq_disable(struct ubase_dev *udev); @@ -94,6 +114,7 @@ void ubase_ctrlq_disable(struct ubase_dev *udev); int __ubase_ctrlq_send(struct ubase_dev *udev, struct ubase_ctrlq_msg *msg, struct ubase_ctrlq_ue_info *ue_info); +bool ubase_ctrlq_check_seq(struct ubase_dev *udev, u16 seq); void ubase_ctrlq_service_task(struct ubase_delay_work *ubase_work); void ubase_ctrlq_handle_crq_msg(struct ubase_dev *udev, struct ubase_ctrlq_base_block *head, diff --git a/drivers/ub/ubase/ubase_dev.c b/drivers/ub/ubase/ubase_dev.c index abdb3232edeabc3dd9743de91068413938775f15..d48582bf90a9f50f02f4f8f8b14bd8a398e7999c 100644 --- a/drivers/ub/ubase/ubase_dev.c +++ b/drivers/ub/ubase/ubase_dev.c @@ -16,6 +16,7 @@ #include "ubase_mailbox.h" #include "ubase_pmem.h" #include "ubase_reset.h" +#include "ubase_stats.h" #include "ubase_dev.h" #define UBASE_PERIOD_100MS 100 @@ -453,21 +454,19 @@ static int ubase_handle_ue2ue_ctrlq_req(struct ubase_dev *udev, return -EINVAL; } - if (cmd->in_size > (len - (sizeof(*cmd) + UBASE_CTRLQ_HDR_LEN))) { - ubase_err(udev, "ubase e2e cmd len = %u error.\n", cmd->in_size); - return -EINVAL; - } - msg.service_ver = head->service_ver; msg.service_type = head->service_type; msg.opcode = head->opcode; msg.need_resp = cmd->need_resp; msg.is_resp = cmd->is_resp; + msg.is_async = cmd->is_async; msg.resp_seq = cmd->seq; msg.in = (u8 *)head + UBASE_CTRLQ_HDR_LEN; msg.in_size = cmd->in_size; msg.out = NULL; msg.out_size = 0; + if (ubase_ctrlq_msg_is_sync_req(&msg)) + msg.is_async = 1; ue_info.bus_ue_id = le16_to_cpu(cmd->head.bus_ue_id); ue_info.seq = cmd->seq; @@ -496,6 +495,11 @@ static int ubase_handle_ue2ue_ctrlq_event(struct ubase_dev *udev, void *data, if (ubase_dev_ctrlq_supported(udev)) return ubase_handle_ue2ue_ctrlq_req(udev, cmd, len); + if (!ubase_ctrlq_check_seq(udev, cmd->seq)) { + ubase_err(udev, "invalid ue2ue ctrlq seq(%u).\n", cmd->seq); + return -EINVAL; + } + head = (struct ubase_ctrlq_base_block *)(cmd + 1); data_len = len - sizeof(*cmd) - UBASE_CTRLQ_HDR_LEN; ubase_ctrlq_handle_crq_msg(udev, head, cmd->seq, @@ -620,6 +624,19 @@ static int ubase_register_cmdq_crq_event(struct ubase_dev *udev) return ret; } +static int ubase_notify_drv_capbilities(struct ubase_dev *udev) +{ + struct ubase_notify_drv_cap_cmd req = {0}; + struct ubase_cmd_buf in; + + set_bit(UBASE_CAP_SUP_ACTIVATE_B, (unsigned long *)req.cap_bits); + + __ubase_fill_inout_buf(&in, UBASE_OPC_NOTIFY_DRV_CAPS, false, + sizeof(req), &req); + + return __ubase_cmd_send_in(udev, &in); +} + static const struct ubase_init_function ubase_init_func_map[] = { { "init work queue", UBASE_SUP_ALL, 0, @@ -629,6 +646,10 @@ static const struct ubase_init_function ubase_init_func_map[] = { "init cmd queue", UBASE_SUP_ALL, 1, ubase_cmd_init, ubase_cmd_uninit }, + { + "notify drv capbilities", UBASE_SUP_ALL, 0, + ubase_notify_drv_capbilities, NULL + }, { "query dev res", UBASE_SUP_ALL, 0, ubase_query_dev_res, NULL @@ -861,6 +882,15 @@ void ubase_resume_aux_devices(struct ubase_dev *udev) mutex_unlock(&priv->uadev_lock); } +/** + * ubase_adev_ubl_supported() - determine whether ub link is supported + * @adev: auxiliary device + * + * This function is used to determine whether ub link is supported. + * + * Context: Any context. + * Return: true or false + */ bool ubase_adev_ubl_supported(struct auxiliary_device *adev) { if (!adev) @@ -870,6 +900,15 @@ bool ubase_adev_ubl_supported(struct auxiliary_device *adev) } EXPORT_SYMBOL(ubase_adev_ubl_supported); +/** + * ubase_adev_ctrlq_supported() - determine whether to support ctrlq + * @adev: auxiliary device + * + * This function is used to determine whether to support ctrlq. + * + * Context: Any context. + * Return: true or false + */ bool ubase_adev_ctrlq_supported(struct auxiliary_device *adev) { if (!adev) @@ -879,6 +918,15 @@ bool ubase_adev_ctrlq_supported(struct auxiliary_device *adev) } EXPORT_SYMBOL(ubase_adev_ctrlq_supported); +/** + * ubase_adev_eth_mac_supported() - determine whether eth link is supported + * @adev: auxiliary device + * + * This function is used to determine whether eth link is supported. + * + * Context: Any context. + * Return: true or false + */ bool ubase_adev_eth_mac_supported(struct auxiliary_device *adev) { if (!adev) @@ -888,6 +936,15 @@ bool ubase_adev_eth_mac_supported(struct auxiliary_device *adev) } EXPORT_SYMBOL(ubase_adev_eth_mac_supported); +/** + * ubase_get_io_base() - get io space base address + * @adev: auxiliary device + * + * The function is used to get io space base address. + * + * Context: Any context. + * Return: NULL if the adev is empty, otherwise the pointer to struct ubase_resource_space + */ struct ubase_resource_space *ubase_get_io_base(struct auxiliary_device *adev) { if (!adev) @@ -897,6 +954,15 @@ struct ubase_resource_space *ubase_get_io_base(struct auxiliary_device *adev) } EXPORT_SYMBOL(ubase_get_io_base); +/** + * ubase_get_mem_base() - get memory space base address + * @adev: auxiliary device + * + * The function is used to get memory space base address. + * + * Context: Any context. + * Return: NULL if the adev is empty, otherwise the pointer to struct ubase_resource_space + */ struct ubase_resource_space *ubase_get_mem_base(struct auxiliary_device *adev) { if (!adev) @@ -906,6 +972,15 @@ struct ubase_resource_space *ubase_get_mem_base(struct auxiliary_device *adev) } EXPORT_SYMBOL(ubase_get_mem_base); +/** + * ubase_get_dev_caps() - get ubase capabilities + * @adev: auxiliary device + * + * The function is used to get ubase capabilities. + * + * Context: Any context. + * Return: NULL if the adev is empty, otherwise the pointer to struct ubase_caps + */ struct ubase_caps *ubase_get_dev_caps(struct auxiliary_device *adev) { if (!adev) @@ -915,6 +990,15 @@ struct ubase_caps *ubase_get_dev_caps(struct auxiliary_device *adev) } EXPORT_SYMBOL(ubase_get_dev_caps); +/** + * ubase_get_udma_caps() - get udma auxiliary device capabilities + * @adev: udma auxiliary device pointer + * + * The function is used to get udma auxiliary device capabilities. + * + * Context: Any context. + * Return: NULL if the adev is empty, otherwise the pointer to struct ubase_adev_caps + */ struct ubase_adev_caps *ubase_get_udma_caps(struct auxiliary_device *adev) { struct ubase_dev *udev; @@ -928,12 +1012,30 @@ struct ubase_adev_caps *ubase_get_udma_caps(struct auxiliary_device *adev) } EXPORT_SYMBOL(ubase_get_udma_caps); +/** + * ubase_get_cdma_caps() - get cdma auxiliary device capabilities + * @adev: cdma auxiliary device pointer + * + * The function is used to get cdma auxiliary device capabilities. + * + * Context: Any context. + * Return: NULL if the adev is empty, otherwise the pointer to struct ubase_adev_caps + */ struct ubase_adev_caps *ubase_get_cdma_caps(struct auxiliary_device *adev) { return ubase_get_udma_caps(adev); } EXPORT_SYMBOL(ubase_get_cdma_caps); +/** + * ubase_get_reset_stage() - get current reset stage + * @adev: auxiliary device + * + * The function is used to get current reset stage. + * + * Context: Any context. + * Return: enum ubase_reset_stage + */ enum ubase_reset_stage ubase_get_reset_stage(struct auxiliary_device *adev) { struct ubase_dev *udev; @@ -947,6 +1049,17 @@ enum ubase_reset_stage ubase_get_reset_stage(struct auxiliary_device *adev) } EXPORT_SYMBOL(ubase_get_reset_stage); +/** + * ubase_virt_register() - register auxiliary device virtualization handling function + * @adev: auxiliary device + * @virt_handler: the function pointer to handle virtualization. adev: the same as the + * parameter 'adev', bus_ue_id: bus ub entity id, is_en: true - enable virtualization, + * false - disable virtualization. + * + * The function is used to register auxiliary device virtualization handling function. + * + * Context: Process context. Takes and releases . + */ void ubase_virt_register(struct auxiliary_device *adev, void (*virt_handler)(struct auxiliary_device *adev, u16 bus_ue_id, bool is_en)) @@ -965,6 +1078,14 @@ void ubase_virt_register(struct auxiliary_device *adev, } EXPORT_SYMBOL(ubase_virt_register); +/** + * ubase_virt_unregister() - unregister auxiliary device virtualization handling function + * @adev: auxiliary device + * + * The function is used to unregister auxiliary device virtualization handling function. + * + * Context: Process context. Takes and releases . + */ void ubase_virt_unregister(struct auxiliary_device *adev) { struct ubase_adev *uadev; @@ -980,6 +1101,16 @@ void ubase_virt_unregister(struct auxiliary_device *adev) } EXPORT_SYMBOL(ubase_virt_unregister); +/** + * ubase_port_register() - register auxiliary device port handling function + * @adev: auxiliary device + * @port_handler: the function pointer to port handling. adev: the same as the + * parameter 'adev', link_up: true - link up, false - link down. + * + * The function is used to register auxiliary device port handling function. + * + * Context: Process context. Takes and releases . + */ void ubase_port_register(struct auxiliary_device *adev, void (*port_handler)(struct auxiliary_device *adev, bool link_up)) @@ -998,6 +1129,14 @@ void ubase_port_register(struct auxiliary_device *adev, } EXPORT_SYMBOL(ubase_port_register); +/** + * ubase_port_unregister() - unregister auxiliary device port handling function + * @adev: auxiliary device + * + * The function is used to unregister auxiliary device port handling function. + * + * Context: Process context. Takes and releases . + */ void ubase_port_unregister(struct auxiliary_device *adev) { struct ubase_adev *uadev; @@ -1013,6 +1152,16 @@ void ubase_port_unregister(struct auxiliary_device *adev) } EXPORT_SYMBOL(ubase_port_unregister); +/** + * ubase_reset_register() - register auxiliary device reset function + * @adev: auxiliary device + * @reset_handler: the function pointer to reset. adev: the same as the parameter + * 'adev', stage: enum ubase_reset_stage. + * + * The function is used to register auxiliary device reset function. + * + * Context: Process context. Takes and releases . + */ void ubase_reset_register(struct auxiliary_device *adev, void (*reset_handler)(struct auxiliary_device *adev, enum ubase_reset_stage stage)) @@ -1031,6 +1180,14 @@ void ubase_reset_register(struct auxiliary_device *adev, } EXPORT_SYMBOL(ubase_reset_register); +/** + * ubase_reset_unregister() - unregister auxiliary device reset function + * @adev: auxiliary device + * + * The function is used to unregister auxiliary device reset function. + * + * Context: Process context. Takes and releases . + */ void ubase_reset_unregister(struct auxiliary_device *adev) { struct ubase_adev *uadev; @@ -1046,6 +1203,15 @@ void ubase_reset_unregister(struct auxiliary_device *adev) } EXPORT_SYMBOL(ubase_reset_unregister); +/** + * ubase_get_unic_caps() - get unic auxiliary device capabilities + * @adev: unic auxiliary device pointer + * + * The function is used to get unic auxiliary device capabilities. + * + * Context: Any context. + * Return: NULL if the adev is empty, otherwise the pointer to struct ubase_adev_caps + */ struct ubase_adev_caps *ubase_get_unic_caps(struct auxiliary_device *adev) { struct ubase_dev *udev; @@ -1141,6 +1307,15 @@ bool ubase_dbg_default(void) return ubase_debug; } +/** + * ubase_get_adev_qos() - get auxiliary device qos information + * @adev: auxiliary device + * + * The function is used to get auxiliary device qos information. + * + * Context: Any context. + * Return: NULL if the adev is empty, otherwise the pointer to struct ubase_adev_qos + */ struct ubase_adev_qos *ubase_get_adev_qos(struct auxiliary_device *adev) { struct ubase_dev *udev; @@ -1178,6 +1353,16 @@ static void ubase_activate_notify(struct ubase_dev *udev, mutex_unlock(&udev->priv.uadev_lock); } +/** + * ubase_activate_register() - register auxiliary device activate handling function + * @adev: auxiliary device + * @activate_handler: the function pointer to activate handling. adev: the same + * as the parameter 'adev', activate: true - activate, false - deactivate. + * + * The function is used to register auxiliary device activate handling function. + * + * Context: Process context. Takes and releases . + */ void ubase_activate_register(struct auxiliary_device *adev, void (*activate_handler)(struct auxiliary_device *adev, bool activate)) @@ -1196,6 +1381,14 @@ void ubase_activate_register(struct auxiliary_device *adev, } EXPORT_SYMBOL(ubase_activate_register); +/** + * ubase_activate_unregister() - unregister auxiliary device activate handling function + * @adev: auxiliary device + * + * The function is used to unregister auxiliary device activate handling function. + * + * Context: Process context. Takes and releases . + */ void ubase_activate_unregister(struct auxiliary_device *adev) { struct ubase_adev *uadev; @@ -1296,6 +1489,25 @@ int ubase_deactivate_handler(struct ubase_dev *udev, u32 bus_ue_id) return ubase_send_activate_dev_req(udev, false, (u16)bus_ue_id); } +void ubase_flush_workqueue(struct ubase_dev *udev) +{ + flush_workqueue(udev->ubase_wq); + flush_workqueue(udev->ubase_async_wq); + flush_workqueue(udev->ubase_period_wq); + flush_workqueue(udev->ubase_arq_wq); +} + +/** + * ubase_activate_dev() - activate device + * @adev: auxiliary device + * + * The auxiliary device actively initializes the activate device process. + * This function will call the activate handling functions registered by all + * auxiliary devices under the same ub entity. + * + * Context: Process context. Takes and releases , BH-safe. + * Return: 0 on success, negative error code otherwise + */ int ubase_activate_dev(struct auxiliary_device *adev) { struct ubase_dev *udev; @@ -1317,15 +1529,29 @@ int ubase_activate_dev(struct auxiliary_device *adev) if (ret) { ubase_err(udev, "failed to activate ubase dev, ret = %d.\n", ret); - return ret; + goto activate_dev_err; } ubase_activate_notify(udev, adev, true); - return 0; +activate_dev_err: + ubase_update_activate_stats(udev, true, ret); + + return ret; } EXPORT_SYMBOL(ubase_activate_dev); +/** + * ubase_deactivate_dev() - deactivate device + * @adev: auxiliary device + * + * The auxiliary device actively initializes the deactivate device process. + * This function will call the activate handling functions registered by all + * auxiliary devices under the same ub entity. + * + * Context: Process context. Takes and releases , BH-safe. + * Return: 0 on success, negative error code otherwise + */ int ubase_deactivate_dev(struct auxiliary_device *adev) { struct ubase_dev *udev; @@ -1352,6 +1578,8 @@ int ubase_deactivate_dev(struct auxiliary_device *adev) ubase_activate_notify(udev, adev, true); } + ubase_update_activate_stats(udev, false, ret); + return ret; } EXPORT_SYMBOL(ubase_deactivate_dev); @@ -1383,6 +1611,16 @@ static int __ubase_get_bus_eid(struct ubase_dev *udev, struct ubase_bus_eid *eid return ubase_query_bus_eid(udev, eid); } +/** + * ubase_get_bus_eid() - get bus entity id + * @adev: auxiliary device + * @eid: save the bus entity id + * + * The function is used to get bus entity id. + * + * Context: Process context. Takes and releases , BH-safe. + * Return: 0 on success, negative error code otherwise + */ int ubase_get_bus_eid(struct auxiliary_device *adev, struct ubase_bus_eid *eid) { struct ubase_dev *udev; diff --git a/drivers/ub/ubase/ubase_dev.h b/drivers/ub/ubase/ubase_dev.h index d32d9fb983776b91679c80b712c0d0f46b6d9dfa..45605409de6b1c8b6f68dbd6ccb085186c3dbac4 100644 --- a/drivers/ub/ubase/ubase_dev.h +++ b/drivers/ub/ubase/ubase_dev.h @@ -178,15 +178,11 @@ struct ubase_ctrlq_msg_ctx { struct completion done; }; -struct ubase_ctrlq_crq_event_nbs { - struct list_head list; - struct ubase_ctrlq_event_nb crq_nb; -}; - struct ubase_ctrlq_crq_table { struct mutex lock; unsigned long last_crq_scheduled; - struct ubase_ctrlq_crq_event_nbs crq_nbs; + u16 crq_nb_cnt; + struct ubase_ctrlq_event_nb *crq_nbs; }; struct ubase_ctrlq { @@ -429,7 +425,6 @@ static inline u32 ubase_ta_timer_align_size(struct ubase_dev *udev) static inline bool ubase_mbx_ue_id_is_valid(u16 mbx_ue_id, struct ubase_dev *udev) { - if (!mbx_ue_id || (mbx_ue_id > udev->caps.dev_caps.ue_num - 1)) return false; @@ -456,4 +451,6 @@ void ubase_virt_handler(struct ubase_dev *udev, u16 bus_ue_id, bool is_en); int ubase_activate_handler(struct ubase_dev *udev, u32 bus_ue_id); int ubase_deactivate_handler(struct ubase_dev *udev, u32 bus_ue_id); +void ubase_flush_workqueue(struct ubase_dev *udev); + #endif diff --git a/drivers/ub/ubase/ubase_eq.c b/drivers/ub/ubase/ubase_eq.c index 754f62011420d467d5021593e095214f17788e97..db1dc72f2caf9967179d72dd65668452599bc0a0 100644 --- a/drivers/ub/ubase/ubase_eq.c +++ b/drivers/ub/ubase/ubase_eq.c @@ -1003,6 +1003,19 @@ static int __ubase_event_register(struct ubase_dev *udev, return ret; } +/** + * ubase_event_register() - register asynchronous event processing function + * @adev: auxiliary device + * @cb: asynchronous event notification block + * + * This function uses `blocking_notifier_chain_register` to register the + * asynchronous event handling function. When the ubase driver receives an + * asynchronous event and matches it with the registered event notification + * block, it calls the registered function via `blocking_notifier_call_chain`. + * + * Context: Process context, Takes and releases the RCU lock. + * Return: 0 on success, negative error code otherwise + */ int ubase_event_register(struct auxiliary_device *adev, struct ubase_event_nb *cb) { @@ -1038,6 +1051,16 @@ static void __ubase_event_unregister(struct ubase_dev *udev, cb->event_type, ret); } +/** + * ubase_event_unregister() - unregister asynchronous event processing function + * @adev: auxiliary device + * @cb: ubase asynchronous event notification block + * + * This function uses `blocking_notifier_chain_unregister` to unregister the + * asynchronous event handling function. + * + * Context: Process context, Takes and releases the RCU lock. + */ void ubase_event_unregister(struct auxiliary_device *adev, struct ubase_event_nb *cb) { @@ -1048,6 +1071,20 @@ void ubase_event_unregister(struct auxiliary_device *adev, } EXPORT_SYMBOL(ubase_event_unregister); +/** + * ubase_comp_register() - register completion event processing function + * @adev: auxiliary device + * @comp_handler: completion event processing function. nb: struct notifier_block, + * jfcn: jfc index, data: self-defined data pointer. + * + * This function uses `atomic_notifier_chain_register` to register the + * completion event handling function. When the ubase driver receives a + * completion event that matches a registered auxiliary device, it calls the + * registered function via `atomic_notifier_call_chain`. + * + * Context: Process context, may sleep + * Return: 0 on success, negative error code otherwise + */ int ubase_comp_register(struct auxiliary_device *adev, int (*comp_handler)(struct notifier_block *nb, unsigned long jfcn, void *data)) @@ -1071,6 +1108,15 @@ int ubase_comp_register(struct auxiliary_device *adev, } EXPORT_SYMBOL(ubase_comp_register); +/** + * ubase_comp_unregister() - unregister completion event processing function + * @adev: auxiliary device + * + * This function uses `atomic_notifier_chain_unregister` to unregister the + * completion event handling function. + * + * Context: Process context, Takes and releases the RCU lock. + */ void ubase_comp_unregister(struct auxiliary_device *adev) { struct ubase_adev *uadev; diff --git a/drivers/ub/ubase/ubase_err_handle.c b/drivers/ub/ubase/ubase_err_handle.c index d91ab0f80b6b4f430907aa5205c942afbbdf2d49..179e59acec0ad4eca358bc6d68e12ec4ebcdfaa4 100644 --- a/drivers/ub/ubase/ubase_err_handle.c +++ b/drivers/ub/ubase/ubase_err_handle.c @@ -9,6 +9,19 @@ #include "ubase_reset.h" #include "ubase_err_handle.h" +static void ubase_notify_himac_reset(struct ubase_dev *udev) +{ + struct ubase_cmd_buf in; + int ret; + + __ubase_fill_inout_buf(&in, UBASE_OPC_HIMAC_RESET, false, 0, NULL); + + ret = __ubase_cmd_send_in(udev, &in); + if (ret) + ubase_err(udev, + "failed to send himac reset cmd, ret = %d.\n", ret); +} + void ubase_errhandle_service_task(struct ubase_delay_work *ubase_work) { struct ubase_dev *udev; @@ -23,6 +36,13 @@ void ubase_errhandle_service_task(struct ubase_delay_work *ubase_work) return; } + if (test_and_clear_bit(UBASE_STATE_HIMAC_RESETTING_B, &udev->state_bits)) { + ubase_err(udev, "ras occurred, ubase need to reset himac.\n"); + ubase_notify_himac_reset(udev); + udev->reset_stat.himac_reset_cnt++; + return; + } + if (test_and_clear_bit(UBASE_STATE_PORT_RESETTING_B, &udev->state_bits)) { ubase_err(udev, "ras occurred, ubase need to reset port.\n"); ubase_port_reset(udev); diff --git a/drivers/ub/ubase/ubase_hw.c b/drivers/ub/ubase/ubase_hw.c index d728dd47f116d0b3629777c789211c9d681732fc..2b68ba5884f51ca6bc815b51f4a41e86b53b7cf9 100644 --- a/drivers/ub/ubase/ubase_hw.c +++ b/drivers/ub/ubase/ubase_hw.c @@ -156,13 +156,10 @@ static void ubase_parse_dev_caps_unic(struct ubase_dev *udev, struct ubase_adev_caps *unic_caps = &udev->caps.unic_caps; unic_caps->jfs.max_cnt = le32_to_cpu(resp->nic_jfs_max_cnt); - unic_caps->jfs.reserved_cnt = le32_to_cpu(resp->nic_jfs_reserved_cnt); unic_caps->jfs.depth = le32_to_cpu(resp->nic_jfs_depth); unic_caps->jfr.max_cnt = le32_to_cpu(resp->nic_jfr_max_cnt); - unic_caps->jfr.reserved_cnt = le32_to_cpu(resp->nic_jfr_reserved_cnt); unic_caps->jfr.depth = le32_to_cpu(resp->nic_jfr_depth); unic_caps->jfc.max_cnt = le32_to_cpu(resp->nic_jfc_max_cnt); - unic_caps->jfc.reserved_cnt = le32_to_cpu(resp->nic_jfc_reserved_cnt); unic_caps->jfc.depth = le32_to_cpu(resp->nic_jfc_depth); unic_caps->cqe_size = le16_to_cpu(resp->nic_cqe_size); unic_caps->utp_port_bitmap = le32_to_cpu(resp->port_bitmap); @@ -174,13 +171,10 @@ static void ubase_parse_dev_caps_udma(struct ubase_dev *udev, struct ubase_adev_caps *udma_caps = &udev->caps.udma_caps; udma_caps->jfs.max_cnt = le32_to_cpu(resp->udma_jfs_max_cnt); - udma_caps->jfs.reserved_cnt = le32_to_cpu(resp->udma_jfs_reserved_cnt); udma_caps->jfs.depth = le32_to_cpu(resp->udma_jfs_depth); udma_caps->jfr.max_cnt = le32_to_cpu(resp->udma_jfr_max_cnt); - udma_caps->jfr.reserved_cnt = le32_to_cpu(resp->udma_jfr_reserved_cnt); udma_caps->jfr.depth = le32_to_cpu(resp->udma_jfr_depth); udma_caps->jfc.max_cnt = le32_to_cpu(resp->udma_jfc_max_cnt); - udma_caps->jfc.reserved_cnt = le32_to_cpu(resp->udma_jfc_reserved_cnt); udma_caps->jfc.depth = le32_to_cpu(resp->udma_jfc_depth); udma_caps->cqe_size = le16_to_cpu(resp->udma_cqe_size); udma_caps->jtg_max_cnt = le32_to_cpu(resp->jtg_max_cnt); @@ -951,7 +945,7 @@ int __ubase_perf_stats(struct ubase_dev *udev, u64 port_bitmap, u32 period, struct ubase_perf_stats_result *data, u32 data_size) { #define UBASE_MS_TO_US(ms) (1000 * (ms)) - struct ubase_stop_perf_stats_cmd resp = {0}; + struct ubase_stop_perf_stats_cmd resp; unsigned long logic_port_bitmap; int ret, j, k, port_num; u8 i; @@ -988,6 +982,8 @@ int __ubase_perf_stats(struct ubase_dev *udev, u64 port_bitmap, u32 period, for (i = 0, k = 0; i < UBASE_MAX_PORT_NUM && k < port_num; i++) { if (!test_bit(i, (unsigned long *)&port_bitmap)) continue; + + memset(&resp, 0, sizeof(resp)); ret = ubase_stop_perf_stats(udev, &resp, period, i); if (ret) goto unlock; @@ -1011,6 +1007,22 @@ int __ubase_perf_stats(struct ubase_dev *udev, u64 port_bitmap, u32 period, return ret; } +/** + * ubase_perf_stats() - get ub port stats + * @adev: auxiliary device + * @port_bitmap: port bitmap + * @period: period, unit: ms + * @data: stats data + * @data_size: data size + * + * The function is used to query the port bandwidth and the bandwidth of each vl + * under the port. The bandwidth statistics collection duration is 'period'. + * The larger the 'period', the longer the time required, and the more accurate + * the bandwidth measurement. + * + * Context: Process context. Takes and releases , BH-safe. Sleep. + * Return: 0 on success, negative error code otherwise + */ int ubase_perf_stats(struct auxiliary_device *adev, u64 port_bitmap, u32 period, struct ubase_perf_stats_result *data, u32 data_size) { diff --git a/drivers/ub/ubase/ubase_hw.h b/drivers/ub/ubase/ubase_hw.h index ee455ba317d1e73c01221abbb9eb27da9e57093d..12bc101cd04fd3be5ef0ce3c33c46e41394f90e2 100644 --- a/drivers/ub/ubase/ubase_hw.h +++ b/drivers/ub/ubase/ubase_hw.h @@ -30,7 +30,7 @@ struct ubase_caps_item { struct ubase_res_cmd_resp { __le32 cap_bits[UBASE_CAP_LEN]; - __le32 rsvd[3]; + __le32 rsvd0[3]; u8 rsvd1[2]; __le16 ceq_vector_num; @@ -43,37 +43,32 @@ struct ubase_res_cmd_resp { __le32 aeqe_depth; __le32 ceqe_depth; __le32 udma_jfs_max_cnt; - __le32 udma_jfs_reserved_cnt; + u8 rsvd2[4]; __le32 udma_jfs_depth; __le32 udma_jfr_max_cnt; - __le32 udma_jfr_reserved_cnt; + u8 rsvd3[4]; __le32 udma_jfr_depth; u8 nic_vl_num; - u8 rsvd2[3]; + u8 rsvd4[3]; u8 nic_vl[UBASE_MAX_REQ_VL_NUM]; __le32 udma_jfc_max_cnt; - __le32 udma_jfc_reserved_cnt; + u8 rsvd5[4]; __le32 udma_jfc_depth; - __le32 udma_tp_max_cnt; - __le32 udma_tp_reserved_cnt; - __le32 udma_tp_depth; - __le32 udma_tpg_max_cnt; - __le32 udma_tpg_reserved_cnt; - __le32 udma_tpg_depth; + u8 rsvd6[24]; __le32 nic_jfs_max_cnt; - __le32 nic_jfs_reserved_cnt; + u8 rsvd7[4]; __le32 nic_jfs_depth; __le32 nic_jfr_max_cnt; - __le32 nic_jfr_reserved_cnt; + u8 rsvd8[4]; __le32 nic_jfr_depth; - __le32 rsvd3[2]; + __le32 rsvd9[2]; - __le32 rsvd4; + __le32 rsvd10; __le32 nic_jfc_max_cnt; - __le32 nic_jfc_reserved_cnt; + u8 rsvd11[4]; __le32 nic_jfc_depth; __le32 nic_tp_max_cnt; __le32 nic_tp_reserved_cnt; @@ -83,10 +78,7 @@ struct ubase_res_cmd_resp { __le32 nic_tpg_reserved_cnt; __le32 nic_tpg_depth; __le32 total_ue_num; - __le32 jfs_ctx_size; - __le32 jfr_ctx_size; - __le32 jfc_ctx_size; - __le32 tp_ctx_size; + u8 rsvd12[16]; __le16 rsvd_jetty_cnt; __le16 mac_stats_num; @@ -95,24 +87,22 @@ struct ubase_res_cmd_resp { __le32 public_jetty_cnt; __le32 tp_extdb_buf_size; __le32 tp_timer_buf_size; - u8 port_work_mode; + u8 resv13; u8 udma_vl_num; u8 udma_tp_resp_vl_offset; u8 ue_num; __le32 port_bitmap; - u8 rsvd5[4]; + u8 rsvd14[4]; /* include udma tp and ctp req vl */ u8 udma_req_vl[UBASE_MAX_REQ_VL_NUM]; __le32 udma_rc_depth; - u8 rsvd6[4]; + u8 rsvd15[4]; __le32 jtg_max_cnt; __le32 rc_max_cnt_per_vl; - __le32 dest_addr_max_cnt; - __le32 seid_upi_max_cnt; + u8 rsvd16[8]; - __le32 tpm_max_cnt; - __le32 ccc_max_cnt; + u8 rsvd17[32]; }; struct ubase_query_oor_resp { diff --git a/drivers/ub/ubase/ubase_mailbox.c b/drivers/ub/ubase/ubase_mailbox.c index 9951929a7f8d5d0ab71d8b1f9617d3f1da013455..86da27cdf8288af2a194e77daeb63e6bf8b330a8 100644 --- a/drivers/ub/ubase/ubase_mailbox.c +++ b/drivers/ub/ubase/ubase_mailbox.c @@ -66,6 +66,15 @@ struct ubase_cmd_mailbox *__ubase_alloc_cmd_mailbox(struct ubase_dev *udev) return NULL; } +/** + * ubase_alloc_cmd_mailbox() - Alloc mailbox buffer + * @aux_dev: auxiliary device + * + * The function is used to alloc mailbox buffer. + * + * Context: Process context. + * Return: NULL if the adev is empty, otherwise the pointer to struct ubase_cmd_mailbox + */ struct ubase_cmd_mailbox *ubase_alloc_cmd_mailbox(struct auxiliary_device *aux_dev) { struct ubase_dev *udev; @@ -91,6 +100,15 @@ void __ubase_free_cmd_mailbox(struct ubase_dev *udev, kfree(mailbox); } +/** + * ubase_free_cmd_mailbox() - Free mailbox buffer + * @aux_dev: auxiliary device + * @mailbox: mailbox command address + * + * The function is used to free mailbox buffer. + * + * Context: Process context. + */ void ubase_free_cmd_mailbox(struct auxiliary_device *aux_dev, struct ubase_cmd_mailbox *mailbox) { @@ -524,6 +542,17 @@ int __ubase_hw_upgrade_ctx_ex(struct ubase_dev *udev, return ret; } +/** + * ubase_hw_upgrade_ctx_ex() - upgrade hardware context + * @aux_dev: auxiliary device + * @attr: the mailbox attribute pointer + * @mailbox: mailbox command address + * + * The function is used to upgrade hardware context. + * + * Context: Process context. Takes and releases , BH-safe. May sleep + * Return: 0 on success, negative error code otherwise + */ int ubase_hw_upgrade_ctx_ex(struct auxiliary_device *aux_dev, struct ubase_mbx_attr *attr, struct ubase_cmd_mailbox *mailbox) diff --git a/drivers/ub/ubase/ubase_pmem.c b/drivers/ub/ubase/ubase_pmem.c index a43f1c39c0be72afff58d361e8040c1e8403befc..71218dc25d94bb8d4d4a91e43ad7945c0725b099 100644 --- a/drivers/ub/ubase/ubase_pmem.c +++ b/drivers/ub/ubase/ubase_pmem.c @@ -336,6 +336,15 @@ void ubase_prealloc_mem_uninit(struct ubase_dev *udev) ubase_pmem_init_map[i].uninit(udev); } +/** + * ubase_adev_prealloc_supported() - determine whether to prealloc buffer + * @adev: auxiliary device + * + * This function is used to determine whether to prealloc buffer. + * + * Context: Any context. + * Return: true or false + */ bool ubase_adev_prealloc_supported(struct auxiliary_device *adev) { struct ubase_dev *udev; diff --git a/drivers/ub/ubase/ubase_qos_hw.c b/drivers/ub/ubase/ubase_qos_hw.c index 5a5881f79547fe5ffa7a88a790ca6c4d030a7da7..bb1a9db877d8940b15f15cfb9913e93015080841 100644 --- a/drivers/ub/ubase/ubase_qos_hw.c +++ b/drivers/ub/ubase/ubase_qos_hw.c @@ -195,13 +195,10 @@ static int __ubase_config_tm_vl_sch(struct ubase_dev *udev, u16 vl_bitmap, int ret; u8 i; - /* the configuration takes effect for all entities. */ - req.bus_ue_id = cpu_to_le16(USHRT_MAX); - req.vl_bitmap = cpu_to_le16(vl_bitmap); - for (i = 0; i < UBASE_MAX_VL_NUM; i++) tsa_bitmap |= vl_tsa[i] ? 1 << i : 0; + req.vl_bitmap = cpu_to_le16(vl_bitmap); req.vl_tsa = cpu_to_le16(tsa_bitmap); memcpy(req.vl_bw, vl_bw, UBASE_MAX_VL_NUM); @@ -209,7 +206,7 @@ static int __ubase_config_tm_vl_sch(struct ubase_dev *udev, u16 vl_bitmap, sizeof(req), &req); ret = __ubase_cmd_send_in(udev, &in); - if (ret) + if (ret && ret != -EPERM) ubase_err(udev, "failed to config tm vl sch, ret = %d", ret); return ret; @@ -230,7 +227,7 @@ static int __ubase_config_ets_vl_sch(struct ubase_dev *udev, u16 vl_bitmap, &req); ret = __ubase_cmd_send_in(udev, &in); - if (ret) + if (ret && ret != -EPERM) ubase_err(udev, "failed to cfg ets vl sch, ret = %d.", ret); return ret; @@ -409,6 +406,55 @@ int ubase_query_fst_fvt_rqmt(struct ubase_dev *udev, return ret; } +static unsigned long ubase_get_sl_bitmap(struct ubase_dev *udev) +{ + struct ubase_adev_qos *qos = &udev->qos; + unsigned long sl_bitmap = 0; + u8 i; + + for (i = 0; i < qos->nic_sl_num; i++) + sl_bitmap |= 1 << qos->nic_sl[i]; + for (i = 0; i < qos->sl_num; i++) + sl_bitmap |= 1 << qos->sl[i]; + + return sl_bitmap; +} + +static int ubase_check_sl_bitmap(struct ubase_dev *udev, unsigned long sl_bitmap) +{ + unsigned long sl_bitmap_cap; + u8 i; + + sl_bitmap_cap = ubase_get_sl_bitmap(udev); + for (i = 0; i < UBASE_MAX_SL_NUM; i++) { + if (!test_bit(i, &sl_bitmap)) + continue; + if (!test_bit(i, &sl_bitmap_cap)) + return -EINVAL; + } + + return 0; +} + +/** + * ubase_check_qos_sch_param() - check qos schedule parameters + * @adev: auxiliary device + * @vl_bitmap: vl bitmap + * @vl_bw: vl bandwidth weight + * @vl_tsa: vl schedule mode + * @is_ets: is ETS flow control mode + * + * The function is used to check qos schedule parameters + * Obtain valid vls through 'vl_bitmap'. The vl scheduling mode 'vl_tsa' supports + * two types: dwrr and sp. The sum of the vl scheduling weights 'vl_bw' must be + * 100. When 'is_ets' is true, it indicates ETS flow control, and the scheduling + * weight for vls with sp scheduling mode must be 0; when 'is_ets' is false, it + * indicates TM flow control, and the scheduling weight for vls with sp + * scheduling mode cannot be 0. + * + * Context: Process context. Takes and releases , BH-safe. + * Return: 0 on success, negative error code otherwise + */ int ubase_check_qos_sch_param(struct auxiliary_device *adev, u16 vl_bitmap, u8 *vl_bw, u8 *vl_tsa, bool is_ets) { @@ -424,6 +470,20 @@ int ubase_check_qos_sch_param(struct auxiliary_device *adev, u16 vl_bitmap, } EXPORT_SYMBOL(ubase_check_qos_sch_param); +/** + * ubase_config_tm_vl_sch() - configuring TM flow control scheduling + * @adev: auxiliary device + * @vl_bitmap: vl bitmap + * @vl_bw: vl bandwidth weight + * @vl_tsa: vl schedule mode + * + * The function is used to configure TM flow control scheduling. + * Configure the scheduling weight 'vl_bw' and scheduling mode 'vl_tsa' + * corresponding to the valid vl in 'vl_bitmap' to the TM flow control. + * + * Context: Process context. Takes and releases , BH-safe. + * Return: 0 on success, negative error code otherwise + */ int ubase_config_tm_vl_sch(struct auxiliary_device *adev, u16 vl_bitmap, u8 *vl_bw, u8 *vl_tsa) { @@ -438,6 +498,25 @@ int ubase_config_tm_vl_sch(struct auxiliary_device *adev, u16 vl_bitmap, } EXPORT_SYMBOL(ubase_config_tm_vl_sch); +/** + * ubase_set_priqos_info() - set priority qos information + * @dev: device + * @sl_priqos: priority qos + * + * The function is used to set priority qos information. + * Through 'sl_priqos->sl_bitmap', obtain the valid priority sl, use sl as an + * index to get the corresponding bandwidth weight and scheduling mode from + * 'sl_priqos->weight' and 'sl_priqos->ch_mode', and configure them to the hardware. + * Specifically, when 'sl_priqos-> port_bitmap' is 0, it configures the TM flow + * control; when 'port_bitmap' is not 0, it configures the ETS flow control for + * the corresponding port. + * The SP scheduling weight for TM flow control cannot be 0; multiple SP traffic + * flows are scheduled according to their weights. For ETS flow control, the SP + * scheduling weight must be 0. + * + * Context: Process context. Takes and releases , BH-safe. + * Return: 0 on success, negative error code otherwise + */ int ubase_set_priqos_info(struct device *dev, struct ubase_sl_priqos *sl_priqos) { struct ubase_dev *udev; @@ -447,6 +526,9 @@ int ubase_set_priqos_info(struct device *dev, struct ubase_sl_priqos *sl_priqos) udev = dev_get_drvdata(dev); + if (ubase_check_sl_bitmap(udev, sl_priqos->sl_bitmap)) + return -EINVAL; + if (sl_priqos->port_bitmap) return ubase_set_ets_priqos(udev, sl_priqos); @@ -454,15 +536,31 @@ int ubase_set_priqos_info(struct device *dev, struct ubase_sl_priqos *sl_priqos) } EXPORT_SYMBOL(ubase_set_priqos_info); +/** + * ubase_get_priqos_info() - get priority qos information + * @dev: device + * @sl_priqos: save the queried priority QoS information + * + * The function is used to get priority qos information. + * Obtain the priority sl available for the device, as well as the corresponding + * bandwidth weight and scheduling mode. + * When port_bitmap is 0, the obtained values are the bandwidth weight and + * scheduling mode for TM flow control; when port_bitmap is not 0, the obtained + * values are the bandwidth weight and scheduling mode for ETS flow control. + * + * Context: Process context. Takes and releases , BH-safe. + * Return: 0 on success, negative error code otherwise + */ int ubase_get_priqos_info(struct device *dev, struct ubase_sl_priqos *sl_priqos) { struct ubase_dev *udev; - if (!dev || !sl_priqos || !sl_priqos->sl_bitmap) + if (!dev || !sl_priqos) return -EINVAL; udev = dev_get_drvdata(dev); + sl_priqos->sl_bitmap = ubase_get_sl_bitmap(udev); if (sl_priqos->port_bitmap) return ubase_get_ets_priqos(udev, sl_priqos); @@ -1057,6 +1155,17 @@ static bool ubase_is_udma_tp_vl(struct ubase_adev_qos *qos, u8 vl) return false; } +/** + * ubase_update_udma_dscp_vl() - update udma's dscp to vl mapping + * @adev: auxiliary device + * @dscp_vl: dscp to vl mapping + * @dscp_num: dscp number + * + * The function updates the dscp to vl mapping based on 'dscp_vl' and saves it + * to 'udma_dscp_vl' in 'truct ubase_adev_qos'. + * + * Context: Any context. + */ void ubase_update_udma_dscp_vl(struct auxiliary_device *adev, u8 *dscp_vl, u8 dscp_num) { diff --git a/drivers/ub/ubase/ubase_reset.c b/drivers/ub/ubase/ubase_reset.c index bb1c281c02d378912fd58e7141843d266933f867..b2fe6d9ef5599a8289d2af5ea98e90fdc587d50e 100644 --- a/drivers/ub/ubase/ubase_reset.c +++ b/drivers/ub/ubase/ubase_reset.c @@ -86,6 +86,15 @@ void __ubase_reset_event(struct ubase_dev *udev, } } +/** + * ubase_reset_event() - reset event processing + * @adev: auxiliary device + * @reset_type: reset type + * + * The function performs corresponding reset processing based on different 'reset_type'. + * + * Context: Any context. + */ void ubase_reset_event(struct auxiliary_device *adev, enum ubase_reset_type reset_type) { @@ -229,6 +238,7 @@ void ubase_suspend(struct ubase_dev *udev) ubase_ctrlq_disable_remote(udev); ubase_ctrlq_disable(udev); ubase_irq_table_free(udev); + ubase_flush_workqueue(udev); } void ubase_resume(struct ubase_dev *udev) diff --git a/drivers/ub/ubase/ubase_stats.c b/drivers/ub/ubase/ubase_stats.c index b30e839ebb0be38401278ecc630af8ac446bb376..4d6e4678686d9078d61e755da538f3523ee8667b 100644 --- a/drivers/ub/ubase/ubase_stats.c +++ b/drivers/ub/ubase/ubase_stats.c @@ -51,6 +51,17 @@ static int ubase_update_mac_stats(struct ubase_dev *udev, u16 port_id, u64 *data return ret; } +/** + * ubase_get_ub_port_stats() - get ub port stats + * @adev: auxiliary device + * @port_id: port id + * @data: ub date link layer stats + * + * The function is used to get ub port stats. + * + * Context: Process context. Takes and releases , BH-safe. Sleep. + * Return: 0 on success, negative error code otherwise + */ int ubase_get_ub_port_stats(struct auxiliary_device *adev, u16 port_id, struct ubase_ub_dl_stats *data) { @@ -65,3 +76,25 @@ int ubase_get_ub_port_stats(struct auxiliary_device *adev, u16 port_id, sizeof(*data) / sizeof(u64), false); } EXPORT_SYMBOL(ubase_get_ub_port_stats); + +void ubase_update_activate_stats(struct ubase_dev *udev, bool activate, + int result) +{ + struct ubase_activate_dev_stats *record = &udev->stats.activate_record; + u64 idx, total; + + mutex_lock(&record->lock); + + if (activate) + record->act_cnt++; + else + record->deact_cnt++; + + total = record->act_cnt + record->deact_cnt; + idx = (total - 1) % UBASE_ACT_STAT_MAX_NUM; + record->stats[idx].activate = activate; + record->stats[idx].time = ktime_get_real_seconds(); + record->stats[idx].result = result; + + mutex_unlock(&record->lock); +} diff --git a/drivers/ub/ubase/ubase_stats.h b/drivers/ub/ubase/ubase_stats.h index b3f6e5d788cc7875d3532738fdd5b0e2d7ab94f8..a6826dd461c76ffaf03dc4ea78fac6dfe31d8494 100644 --- a/drivers/ub/ubase/ubase_stats.h +++ b/drivers/ub/ubase/ubase_stats.h @@ -15,4 +15,7 @@ struct ubase_query_mac_stats_cmd { __le64 stats_val[]; }; +void ubase_update_activate_stats(struct ubase_dev *udev, bool activate, + int result); + #endif /* _UBASE_STATS_H */ diff --git a/drivers/ub/ubase/ubase_tp.h b/drivers/ub/ubase/ubase_tp.h index 965535a18f1afa502b6b54d52643433df2210eb5..0506e77c98f082e68bd0bc439067120b76403e80 100644 --- a/drivers/ub/ubase/ubase_tp.h +++ b/drivers/ub/ubase/ubase_tp.h @@ -11,8 +11,34 @@ #include "ubase_dev.h" +#define UBASE_TP_PORT_BITMAP_STEP 2 + #define UBASE_WAIT_TP_FLUSH_TOTAL_STEPS 12 +struct ubase_tp_ctx { + u32 rsvd0; + u32 wqe_ba_l; + u32 wqe_ba_h : 20; + u32 rsvd1 : 12; + u32 rsvd2[5]; + u32 rsvd3_0 : 4; + u32 tp_wqe_token_id : 20; + u32 rsvd3_1 : 8; + u32 rsvd4[5]; + u32 rsvd5 : 4; + u32 reorder_q_addr_l : 28; + u32 reorder_q_addr_h : 24; + u32 rsvd6 : 8; + u32 rsvd7[5]; + u32 scc_token : 19; + u32 rsvd8 : 13; + u32 rsvd9[4]; + u32 rsvd10_0 : 24; + u32 scc_token_1 : 4; + u32 rsvd10_1 : 4; + u32 rsvd11[37]; +}; + struct ubase_tpg { u32 mb_tpgn; u8 tpg_state; diff --git a/drivers/ub/ubus/memory.c b/drivers/ub/ubus/memory.c index e7b3144db4bd4974b9d627c51dc5c752b6c3f1d4..e7fcdf76f28e537e7fa238574aae6b603c4c3bd5 100644 --- a/drivers/ub/ubus/memory.c +++ b/drivers/ub/ubus/memory.c @@ -114,7 +114,7 @@ void ub_mem_drain_start(u32 scna) } if (mem_device->ops && mem_device->ops->mem_drain_start) - mem_device->ops->mem_drain_start(mem_device); + mem_device->ops->mem_drain_start(ubc); else dev_warn(mem_device->dev, "ub mem_device ops mem_drain_start is null.\n"); } @@ -138,7 +138,7 @@ int ub_mem_drain_state(u32 scna) } if (mem_device->ops && mem_device->ops->mem_drain_state) - return mem_device->ops->mem_drain_state(mem_device); + return mem_device->ops->mem_drain_state(ubc); dev_warn(mem_device->dev, "ub memory decoder ops mem_drain_state is null.\n"); return 0; diff --git a/drivers/ub/ubus/memory.h b/drivers/ub/ubus/memory.h index 7c841b466f3ef3693b6b061590d0a2ef12694731..f96f7a29061664c81007c79d5c3a778b979591ff 100644 --- a/drivers/ub/ubus/memory.h +++ b/drivers/ub/ubus/memory.h @@ -29,8 +29,8 @@ struct ub_mem_ras_ctx { }; struct ub_mem_device_ops { - void (*mem_drain_start)(struct ub_mem_device *mem_device); - int (*mem_drain_state)(struct ub_mem_device *mem_device); + void (*mem_drain_start)(struct ub_bus_controller *ubc); + int (*mem_drain_state)(struct ub_bus_controller *ubc); bool (*mem_validate_pa)(struct ub_bus_controller *ubc, u64 pa_start, u64 pa_end, bool cacheable); diff --git a/drivers/ub/ubus/pool.c b/drivers/ub/ubus/pool.c index 2aeb8d57ee9aea5461f402b0e6eea1909451c514..e86b19b58f630ef9e4b309f8b26431976a3d54ff 100644 --- a/drivers/ub/ubus/pool.c +++ b/drivers/ub/ubus/pool.c @@ -456,6 +456,18 @@ int ub_fm_flush_ubc_info(struct ub_bus_controller *ubc) if (!buf) goto out; + ret = ub_query_ent_na(ubc->uent, buf); + if (ret) { + dev_err(dev, "update cluster ubc cna failed, ret=%d\n", ret); + goto free_buf; + } + + ret = ub_query_port_na(ubc->uent, buf); + if (ret) { + dev_err(dev, "update cluster ubc port cna failed, ret=%d\n", ret); + goto free_buf; + } + ret = ub_cfg_read_word(ubc->uent, UB_UPI, &upi); if (ret) { dev_err(dev, "update cluster upi failed, ret=%d\n", ret); @@ -489,14 +501,6 @@ int ub_fm_flush_ubc_info(struct ub_bus_controller *ubc) ubc->uent->fm_cna = fm_cna & UB_FM_CNA_MASK; dev_info(dev, "update cluster ubc fm cna to %#x\n", ubc->uent->fm_cna); - ret = ub_query_ent_na(ubc->uent, buf); - if (ret) { - dev_err(dev, "update cluster ubc cna failed, ret=%d\n", ret); - goto free_buf; - } - - ret = ub_query_port_na(ubc->uent, buf); - free_buf: kfree(buf); out: diff --git a/drivers/ub/ubus/ubus.h b/drivers/ub/ubus/ubus.h index a4b46402e32d5168c7c94bbef8583ab2b75d0f39..d26e0816b89bcef816d8aade74a1cea1a69eb477 100644 --- a/drivers/ub/ubus/ubus.h +++ b/drivers/ub/ubus/ubus.h @@ -56,8 +56,6 @@ static inline bool ub_entity_test_priv_flag(struct ub_entity *uent, int bit) return test_bit(bit, &uent->priv_flags); } -int ub_host_probe(void); -void ub_host_remove(void); struct ub_bus_controller *ub_find_bus_controller(u32 ctl_no); struct ub_manage_subsystem_ops { diff --git a/drivers/ub/ubus/ubus_driver.c b/drivers/ub/ubus/ubus_driver.c index b4c6f0d3b760fbac7c589e107ceba5b18a61cf4c..9431bbccd3b0ff6c1358b613bc8077492dffd4a8 100644 --- a/drivers/ub/ubus/ubus_driver.c +++ b/drivers/ub/ubus/ubus_driver.c @@ -33,45 +33,12 @@ MODULE_PARM_DESC(entity_flex_en, "Entity Flexible enable: default: 0"); DECLARE_RWSEM(ub_bus_sem); +#define UBC_GUID_VENDOR_SHIFT 48 +#define UBC_GUID_VENDOR_MASK GENMASK(15, 0) + static DEFINE_MUTEX(manage_subsystem_ops_mutex); static const struct ub_manage_subsystem_ops *manage_subsystem_ops; -int register_ub_manage_subsystem_ops(const struct ub_manage_subsystem_ops *ops) -{ - if (!ops) - return -EINVAL; - - mutex_lock(&manage_subsystem_ops_mutex); - if (!manage_subsystem_ops) { - manage_subsystem_ops = ops; - mutex_unlock(&manage_subsystem_ops_mutex); - pr_info("ub manage subsystem ops register successfully\n"); - return 0; - } - - pr_warn("ub manage subsystem ops has been registered\n"); - mutex_unlock(&manage_subsystem_ops_mutex); - - return -EINVAL; -} -EXPORT_SYMBOL_GPL(register_ub_manage_subsystem_ops); - -void unregister_ub_manage_subsystem_ops(const struct ub_manage_subsystem_ops *ops) -{ - if (!ops) - return; - - mutex_lock(&manage_subsystem_ops_mutex); - if (manage_subsystem_ops == ops) { - manage_subsystem_ops = NULL; - pr_info("ub manage subsystem ops unregister successfully\n"); - } else { - pr_warn("ub manage subsystem ops is not registered by this vendor\n"); - } - mutex_unlock(&manage_subsystem_ops_mutex); -} -EXPORT_SYMBOL_GPL(unregister_ub_manage_subsystem_ops); - const struct ub_manage_subsystem_ops *get_ub_manage_subsystem_ops(void) { return manage_subsystem_ops; @@ -653,7 +620,7 @@ static void ubus_driver_resource_drain(void) ub_static_cluster_instance_drain(); } -int ub_host_probe(void) +static int ub_host_probe(void) { int ret; @@ -724,9 +691,8 @@ int ub_host_probe(void) ub_bus_type_uninit(); return ret; } -EXPORT_SYMBOL_GPL(ub_host_probe); -void ub_host_remove(void) +static void ub_host_remove(void) { message_rx_uninit(); if (manage_subsystem_ops && manage_subsystem_ops->ras_handler_remove) @@ -741,7 +707,61 @@ void ub_host_remove(void) unregister_ub_cfg_ops(); ub_bus_type_uninit(); } -EXPORT_SYMBOL_GPL(ub_host_remove); + +int register_ub_manage_subsystem_ops(const struct ub_manage_subsystem_ops *ops) +{ + struct ub_bus_controller *ubc; + int ret; + + if (!ops) { + pr_err("ub manage subsystem ops is NULL\n"); + return -EINVAL; + } + + mutex_lock(&manage_subsystem_ops_mutex); + if (!manage_subsystem_ops) { + list_for_each_entry(ubc, &ubc_list, node) { + if (((ubc->attr.ubc_guid_high >> UBC_GUID_VENDOR_SHIFT) & + UBC_GUID_VENDOR_MASK) == ops->vendor) { + manage_subsystem_ops = ops; + ret = ub_host_probe(); + if (ret) + manage_subsystem_ops = NULL; + else + pr_info("ub manage subsystem ops register successfully\n"); + + mutex_unlock(&manage_subsystem_ops_mutex); + return ret; + } + } + pr_warn("ub manage subsystem ops is not match with any of ub controller\n"); + } else { + pr_warn("ub manage subsystem ops has been registered\n"); + } + mutex_unlock(&manage_subsystem_ops_mutex); + + return 0; +} +EXPORT_SYMBOL_GPL(register_ub_manage_subsystem_ops); + +void unregister_ub_manage_subsystem_ops(const struct ub_manage_subsystem_ops *ops) +{ + if (!ops) { + pr_err("ub manage subsystem ops is NULL\n"); + return; + } + + mutex_lock(&manage_subsystem_ops_mutex); + if (manage_subsystem_ops == ops) { + ub_host_remove(); + manage_subsystem_ops = NULL; + pr_info("ub manage subsystem ops unregister successfully\n"); + } else { + pr_warn("ub manage subsystem ops is not registered by this vendor\n"); + } + mutex_unlock(&manage_subsystem_ops_mutex); +} +EXPORT_SYMBOL_GPL(unregister_ub_manage_subsystem_ops); static int __init ubus_driver_init(void) { diff --git a/drivers/ub/ubus/ubus_driver.h b/drivers/ub/ubus/ubus_driver.h index f2bff32bbee9c075a3095b15819012f7f2f55320..b2eab906fa31b03872d5706f590cae5739b8bba6 100644 --- a/drivers/ub/ubus/ubus_driver.h +++ b/drivers/ub/ubus/ubus_driver.h @@ -8,7 +8,5 @@ extern struct rw_semaphore ub_bus_sem; extern struct bus_type ub_service_bus_type; -int ub_host_probe(void); -void ub_host_remove(void); #endif /* __UBUS_DRIVER_H__ */ diff --git a/drivers/ub/ubus/vendor/hisilicon/hisi-ubus.c b/drivers/ub/ubus/vendor/hisilicon/hisi-ubus.c index be86f055cb34961c1c9d204c989f4ed63aa81907..3627b0e8f0185df7482217fe1df976f55a23eddd 100644 --- a/drivers/ub/ubus/vendor/hisilicon/hisi-ubus.c +++ b/drivers/ub/ubus/vendor/hisilicon/hisi-ubus.c @@ -53,10 +53,6 @@ static int __init hisi_ubus_driver_register(struct platform_driver *drv) if (ret) return ret; - ret = ub_host_probe(); - if (ret) - goto host_probe_fail; - ret = platform_driver_register(drv); if (ret) goto platform_driver_register_fail; @@ -64,8 +60,6 @@ static int __init hisi_ubus_driver_register(struct platform_driver *drv) return 0; platform_driver_register_fail: - ub_host_remove(); -host_probe_fail: unregister_ub_manage_subsystem_ops(&hisi_ub_manage_subsystem_ops); return ret; } @@ -73,7 +67,6 @@ static int __init hisi_ubus_driver_register(struct platform_driver *drv) static void __exit hisi_ubus_driver_unregister(struct platform_driver *drv) { platform_driver_unregister(drv); - ub_host_remove(); unregister_ub_manage_subsystem_ops(&hisi_ub_manage_subsystem_ops); } diff --git a/drivers/ub/ubus/vendor/hisilicon/hisi-ubus.h b/drivers/ub/ubus/vendor/hisilicon/hisi-ubus.h index 9aa3ba5521c1bd5725eaf5d4e6328cbc7ff9850e..092695e9d43c349054e4827bb188fe23f96843f6 100644 --- a/drivers/ub/ubus/vendor/hisilicon/hisi-ubus.h +++ b/drivers/ub/ubus/vendor/hisilicon/hisi-ubus.h @@ -12,6 +12,10 @@ #define MB_SIZE_OFFSET 20 #define HI_UBC_PRIVATE_DATA_RESERVED 3 #define HI_UBC_PRIVATE_DATA_RESERVED2 111 +#define UB_MEM_VERSION_INVALID 0xffffffff +#define UB_MEM_VERSION_0 0 +#define UB_MEM_VERSION_1 1 +#define UB_MEM_VERSION_2 2 struct hi_mem_pa_info { u64 decode_addr; diff --git a/drivers/ub/ubus/vendor/hisilicon/memory.c b/drivers/ub/ubus/vendor/hisilicon/memory.c index 4d4f80f847fca2456f81f2d3cb6593b9da361ca7..fa9747171eeaa791ef30eb6b112ed032ffdd1924 100644 --- a/drivers/ub/ubus/vendor/hisilicon/memory.c +++ b/drivers/ub/ubus/vendor/hisilicon/memory.c @@ -24,9 +24,13 @@ #define MEM_EVENT_MAX_NUM 16 #define MAR_ERR_ADDR_COUNT 10 #define MAR_ERR_ADDR_SIZE 2 +#define MEM_DECODER_NUMBER_V1 5 +#define MEM_DECODER_NUMBER_V2 2 #define hpa_gen(addr_h, addr_l) (((u64)(addr_h) << 32) | (addr_l)) +static u8 ub_mem_num; + struct ub_mem_decoder { struct device *dev; struct ub_entity *uent; @@ -58,25 +62,26 @@ struct hi_get_ubmem_event_pld { static bool hi_mem_validate_pa(struct ub_bus_controller *ubc, u64 pa_start, u64 pa_end, bool cacheable); -static void hi_mem_drain_start(struct ub_mem_device *mem_device) +static void hi_mem_drain_start(struct ub_bus_controller *ubc) { - struct ub_mem_decoder *decoder, *data = mem_device->priv_data; + struct ub_mem_decoder *decoder, *data = ubc->mem_device->priv_data; if (!data) { - dev_err(mem_device->dev, "ubc mem_decoder is null.\n"); + dev_err(&ubc->dev, "ubc mem_decoder is null.\n"); return; } - for (int i = 0; i < MEM_INFO_NUM; i++) { + for (int i = 0; i < ub_mem_num; i++) { decoder = &data[i]; writel(0, decoder->base_reg + DRAIN_ENABLE_REG_OFFSET); writel(1, decoder->base_reg + DRAIN_ENABLE_REG_OFFSET); } } -static int hi_mem_drain_state(struct ub_mem_device *mem_device) +static int hi_mem_drain_state(struct ub_bus_controller *ubc) { - struct ub_mem_decoder *decoder, *data = mem_device->priv_data; + struct ub_mem_decoder *decoder, *data = ubc->mem_device->priv_data; + struct ub_mem_device *mem_device = ubc->mem_device; int val = 0; if (!data) { @@ -84,7 +89,7 @@ static int hi_mem_drain_state(struct ub_mem_device *mem_device) return 0; } - for (int i = 0; i < MEM_INFO_NUM; i++) { + for (int i = 0; i < ub_mem_num; i++) { decoder = &data[i]; val = readb(decoder->base_reg + DRAIN_STATE_REG_OFFSET) & 0x1; dev_info_ratelimited(decoder->dev, "ub memory decoder[%d] drain state, val=%d\n", @@ -246,16 +251,25 @@ static irqreturn_t hi_mem_ras_irq(int irq, void *context) return IRQ_WAKE_THREAD; } -static int hi_mem_decoder_create_one(struct ub_bus_controller *ubc, int mar_id) +static bool is_ub_mem_version_valid(struct ub_bus_controller *ubc) +{ + struct hi_ubc_private_data *data = ubc->data; + + if (!data || data->ub_mem_version == UB_MEM_VERSION_INVALID) + return false; + return true; +} + +static int hi_mem_decoder_create_one(struct ub_bus_controller *ubc, int index) { - struct hi_ubc_private_data *data = (struct hi_ubc_private_data *)ubc->data; struct ub_mem_decoder *decoder, *priv_data = ubc->mem_device->priv_data; + struct hi_ubc_private_data *data = ubc->data; - decoder = &priv_data[mar_id]; + decoder = &priv_data[index]; decoder->dev = &ubc->dev; decoder->uent = ubc->uent; - decoder->base_reg = ioremap(data->mem_pa_info[mar_id].decode_addr, + decoder->base_reg = ioremap(data->mem_pa_info[index].decode_addr, SZ_64); if (!decoder->base_reg) { dev_err(decoder->dev, "ub mem decoder base reg ioremap failed.\n"); @@ -265,24 +279,47 @@ static int hi_mem_decoder_create_one(struct ub_bus_controller *ubc, int mar_id) return 0; } -static void hi_mem_decoder_remove_one(struct ub_bus_controller *ubc, int mar_id) +static void hi_mem_decoder_remove_one(struct ub_bus_controller *ubc, int index) { struct ub_mem_decoder *priv_data = ubc->mem_device->priv_data; - iounmap(priv_data[mar_id].base_reg); + iounmap(priv_data[index].base_reg); +} + +static u8 get_mem_decoder_number(struct hi_ubc_private_data *data) +{ + switch (data->ub_mem_version) { + case UB_MEM_VERSION_0: + case UB_MEM_VERSION_1: + return MEM_DECODER_NUMBER_V1; + case UB_MEM_VERSION_2: + return MEM_DECODER_NUMBER_V2; + default: + return 0; + } } int hi_mem_decoder_create(struct ub_bus_controller *ubc) { struct ub_mem_device *mem_device; + struct hi_ubc_private_data *data; void *priv_data; int ret; + if (!is_ub_mem_version_valid(ubc)) { + dev_info(&ubc->dev, "Don't need to create mem decoder\n"); + return 0; + } + + ub_mem_num = get_mem_decoder_number(data); + if (!ub_mem_num) + return -EINVAL; + mem_device = kzalloc(sizeof(*mem_device), GFP_KERNEL); if (!mem_device) return -ENOMEM; - priv_data = kcalloc(MEM_INFO_NUM, sizeof(struct ub_mem_decoder), + priv_data = kcalloc(ub_mem_num, sizeof(struct ub_mem_decoder), GFP_KERNEL); if (!priv_data) { kfree(mem_device); @@ -296,7 +333,7 @@ int hi_mem_decoder_create(struct ub_bus_controller *ubc) mem_device->priv_data = priv_data; ubc->mem_device = mem_device; - for (int i = 0; i < MEM_INFO_NUM; i++) { + for (int i = 0; i < ub_mem_num; i++) { ret = hi_mem_decoder_create_one(ubc, i); if (ret) { dev_err(&ubc->dev, "hi mem create decoder %d failed\n", i); @@ -318,7 +355,12 @@ void hi_mem_decoder_remove(struct ub_bus_controller *ubc) if (!ubc->mem_device) return; - for (int i = 0; i < MEM_INFO_NUM; i++) + if (!is_ub_mem_version_valid(ubc)) { + dev_info(&ubc->dev, "Don't need to remove mem decoder\n"); + return; + } + + for (int i = 0; i < ub_mem_num; i++) hi_mem_decoder_remove_one(ubc, i); kfree(ubc->mem_device->priv_data); @@ -333,7 +375,12 @@ void hi_register_ubmem_irq(struct ub_bus_controller *ubc) u32 usi_idx; if (!ubc->mem_device) { - pr_err("mem device is NULL!\n"); + pr_err("register ubmem irq failed, mem device is NULL!\n"); + return; + } + + if (!is_ub_mem_version_valid(ubc)) { + dev_info(&ubc->dev, "Don't need to register_ubmem_irq\n"); return; } @@ -371,6 +418,11 @@ void hi_unregister_ubmem_irq(struct ub_bus_controller *ubc) return; } + if (!is_ub_mem_version_valid(ubc)) { + dev_info(&ubc->dev, "Don't need to unregister_ubmem_irq\n"); + return; + } + irq_num = ubc->mem_device->ubmem_irq_num; if (irq_num < 0) return; @@ -404,8 +456,8 @@ static bool hi_mem_validate_pa(struct ub_bus_controller *ubc, return false; } - data = (struct hi_ubc_private_data *)ubc->data; - for (u16 i = 0; i < MEM_INFO_NUM; i++) { + data = ubc->data; + for (u16 i = 0; i < ub_mem_num; i++) { if (ub_hpa_valid(pa_start, pa_end, data->mem_pa_info[i].cc_base_addr, data->mem_pa_info[i].cc_base_size) && diff --git a/drivers/ub/urma/hw/udma/Kconfig b/drivers/ub/urma/hw/udma/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..c6d5ca89e7ef75c8fa7031e0d602308bee2a1bff --- /dev/null +++ b/drivers/ub/urma/hw/udma/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. + +config UB_UDMA + default n + tristate "UB UDMA Driver" + depends on UB_UBASE && UB_URMA && UB_UMMU_CORE + help + UDMA driver support for Hisilicon UBUS engine + in Hisilicon SoC. To compile this driver, + choose Y here: if UB_UDMA is m, this module + will be called udma. diff --git a/drivers/ub/urma/hw/udma/Makefile b/drivers/ub/urma/hw/udma/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..8eddc0984ac7dc4066cd79ee7112e822b503fae6 --- /dev/null +++ b/drivers/ub/urma/hw/udma/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0+ + +udma-$(CONFIG_UB_UDMA) := udma_main.o udma_cmd.o udma_common.o udma_ctx.o udma_db.o \ + udma_rct.o udma_tid.o udma_debugfs.o udma_eq.o udma_jfc.o \ + udma_ctrlq_tp.o udma_eid.o udma_ctl.o udma_segment.o \ + udma_dfx.o udma_jfs.o udma_jetty.o udma_jfr.o + +obj-m := udma.o diff --git a/drivers/ub/urma/hw/udma/udma_cmd.c b/drivers/ub/urma/hw/udma/udma_cmd.c new file mode 100644 index 0000000000000000000000000000000000000000..0e3c18c81e25c98aac2a9512b6dbd5c122babdc5 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_cmd.c @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt + +#include +#include +#include +#include "udma_eid.h" +#include "udma_cmd.h" +#include "udma_jfc.h" +#include "udma_jfr.h" +#include "udma_jetty.h" + +bool debug_switch = true; + +int udma_cmd_init(struct udma_dev *udma_dev) +{ + sema_init(&udma_dev->mb_cmd.poll_sem, 1); + udma_dev->mb_cmd.pool = dma_pool_create("udma_cmd", udma_dev->dev, + UDMA_MAILBOX_SIZE, + UDMA_MAILBOX_SIZE, 0); + if (!udma_dev->mb_cmd.pool) { + dev_err(udma_dev->dev, "failed to dma_pool_create.\n"); + return -ENOMEM; + } + + init_rwsem(&udma_dev->mb_cmd.udma_mb_rwsem); + + return 0; +} + +void udma_cmd_cleanup(struct udma_dev *udma_dev) +{ + down_write(&udma_dev->mb_cmd.udma_mb_rwsem); + dma_pool_destroy(udma_dev->mb_cmd.pool); + up_write(&udma_dev->mb_cmd.udma_mb_rwsem); +} + +struct ubase_cmd_mailbox *udma_alloc_cmd_mailbox(struct udma_dev *dev) +{ + struct ubase_cmd_mailbox *mailbox; + + mailbox = kzalloc(sizeof(*mailbox), GFP_KERNEL); + if (!mailbox) + goto failed_alloc_mailbox; + + down_read(&dev->mb_cmd.udma_mb_rwsem); + mailbox->buf = dma_pool_zalloc(dev->mb_cmd.pool, GFP_KERNEL, + &mailbox->dma); + if (!mailbox->buf) { + dev_err(dev->dev, "failed to alloc buffer of mailbox.\n"); + goto failed_alloc_mailbox_buf; + } + + return mailbox; + +failed_alloc_mailbox_buf: + up_read(&dev->mb_cmd.udma_mb_rwsem); + kfree(mailbox); +failed_alloc_mailbox: + return NULL; +} + +void udma_free_cmd_mailbox(struct udma_dev *dev, + struct ubase_cmd_mailbox *mailbox) +{ + if (!mailbox) { + dev_err(dev->dev, "Invalid mailbox.\n"); + return; + } + + dma_pool_free(dev->mb_cmd.pool, mailbox->buf, mailbox->dma); + up_read(&dev->mb_cmd.udma_mb_rwsem); + kfree(mailbox); +} + +static void udma_set_mb_flag_or_fd(uint8_t op, struct udma_mbx_op_match *match, + void *buf) +{ + struct udma_jetty_ctx *jfs_ctx; + + if (op == UDMA_CMD_QUERY_JFS_CONTEXT) { + jfs_ctx = (struct udma_jetty_ctx *)buf; + jfs_ctx->flush_cqe_done = 1; + jfs_ctx->state = 1; + jfs_ctx->flush_ssn_vld = 1; + } +} + +static bool udma_op_ignore_eagain(uint8_t op, void *buf) +{ + struct udma_mbx_op_match matches[] = { + { UDMA_CMD_CREATE_JFS_CONTEXT, false }, + { UDMA_CMD_MODIFY_JFS_CONTEXT, true }, + { UDMA_CMD_DESTROY_JFS_CONTEXT, true }, + { UDMA_CMD_QUERY_JFS_CONTEXT, true }, + { UDMA_CMD_CREATE_JFC_CONTEXT, false }, + { UDMA_CMD_MODIFY_JFC_CONTEXT, true }, + { UDMA_CMD_DESTROY_JFC_CONTEXT, true }, + { UDMA_CMD_QUERY_JFC_CONTEXT, true }, + { UDMA_CMD_CREATE_JFR_CONTEXT, false }, + { UDMA_CMD_MODIFY_JFR_CONTEXT, true }, + { UDMA_CMD_DESTROY_JFR_CONTEXT, true }, + { UDMA_CMD_QUERY_JFR_CONTEXT, true }, + { UDMA_CMD_QUERY_TP_CONTEXT, true }, + { UDMA_CMD_CREATE_JETTY_GROUP_CONTEXT, false }, + { UDMA_CMD_MODIFY_JETTY_GROUP_CONTEXT, true }, + { UDMA_CMD_DESTROY_JETTY_GROUP_CONTEXT, true }, + { UDMA_CMD_QUERY_JETTY_GROUP_CONTEXT, true }, + { UDMA_CMD_CREATE_RC_CONTEXT, false }, + { UDMA_CMD_MODIFY_RC_CONTEXT, true }, + { UDMA_CMD_DESTROY_RC_CONTEXT, true }, + { UDMA_CMD_QUERY_RC_CONTEXT, true }, + { UDMA_CMD_READ_SEID_UPI, true }, + }; + uint32_t i; + + for (i = 0; i < ARRAY_SIZE(matches); i++) { + if (op == matches[i].op) { + udma_set_mb_flag_or_fd(op, &matches[i], buf); + return matches[i].ignore_ret; + } + } + + return false; +} + +int udma_post_mbox(struct udma_dev *dev, struct ubase_cmd_mailbox *mailbox, + struct ubase_mbx_attr *attr) +{ + int ret; + + if (debug_switch) + dev_info_ratelimited(dev->dev, + "Send cmd mailbox, data: %08x %04x%04x.\n", + attr->tag, attr->op, attr->mbx_ue_id); + + ret = ubase_hw_upgrade_ctx_ex(dev->comdev.adev, attr, mailbox); + + return (ret == -EAGAIN && + udma_op_ignore_eagain(attr->op, mailbox->buf)) ? 0 : ret; +} + +int udma_config_ctx_buf_to_hw(struct udma_dev *udma_dev, + struct udma_buf *ctx_buf, + struct ubase_mbx_attr *attr) +{ + struct ubase_cmd_mailbox mailbox; + int ret; + + mailbox.dma = ctx_buf->addr; + ret = udma_post_mbox(udma_dev, &mailbox, attr); + if (ret) + dev_err(udma_dev->dev, + "failed to config ctx_buf to hw, ret = %d.\n", ret); + + return ret; +} + +int udma_cmd_query_hw_resource(struct udma_dev *udma_dev, void *out_addr) +{ + struct ubase_cmd_buf out = {}; + struct ubase_cmd_buf in = {}; + + udma_fill_buf(&in, UDMA_CMD_QUERY_UE_RES, true, 0, NULL); + udma_fill_buf(&out, UDMA_CMD_QUERY_UE_RES, true, + sizeof(struct udma_cmd_ue_resource), out_addr); + + return ubase_cmd_send_inout(udma_dev->comdev.adev, &in, &out); +} + +int post_mailbox_update_ctx(struct udma_dev *udma_dev, void *ctx, uint32_t size, + struct ubase_mbx_attr *attr) +{ + struct ubase_cmd_mailbox *mailbox; + int ret; + + mailbox = udma_alloc_cmd_mailbox(udma_dev); + if (!mailbox) { + dev_err(udma_dev->dev, + "failed to alloc mailbox for opcode 0x%x.\n", attr->op); + return -ENOMEM; + } + + if (ctx) + memcpy(mailbox->buf, ctx, size); + + ret = udma_post_mbox(udma_dev, mailbox, attr); + if (ret) + dev_err(udma_dev->dev, + "failed to post mailbox, opcode = 0x%x, ret = %d.\n", attr->op, + ret); + + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +struct ubase_cmd_mailbox *udma_mailbox_query_ctx(struct udma_dev *udma_dev, + struct ubase_mbx_attr *attr) +{ + struct ubase_cmd_mailbox *mailbox; + int ret; + + mailbox = udma_alloc_cmd_mailbox(udma_dev); + if (!mailbox) { + dev_err(udma_dev->dev, + "failed to alloc mailbox query ctx, opcode = %u, id = %u.\n", + attr->op, attr->tag); + return NULL; + } + + ret = udma_post_mbox(udma_dev, mailbox, attr); + if (ret) { + dev_err(udma_dev->dev, + "failed to post mailbox query ctx, opcode = %u, id = %u, ret = %d.\n", + attr->op, attr->tag, ret); + udma_free_cmd_mailbox(udma_dev, mailbox); + return NULL; + } + + return mailbox; +} + +int udma_close_ue_rx(struct udma_dev *dev, bool check_feature_enable, bool check_ta_flush, + bool is_reset, uint32_t tp_num) +{ + int ret = 0; + + if (check_ta_flush) + return ret; + + if (check_feature_enable && !(dev->caps.feature & UDMA_CAP_FEATURE_UE_RX_CLOSE)) + return ret; + + mutex_lock(&dev->disable_ue_rx_mutex); + if (dev->disable_ue_rx_count == 0 && !is_reset) { + ret = ubase_deactivate_dev(dev->comdev.adev); + if (ret) { + dev_err(dev->dev, "failed to close ue rx, ret = %d.\n", ret); + goto out; + } + } + if (tp_num) + dev->disable_ue_rx_count += tp_num; + else + dev->disable_ue_rx_count++; +out: + mutex_unlock(&dev->disable_ue_rx_mutex); + + return ret; +} + +int udma_open_ue_rx(struct udma_dev *dev, bool check_feature_enable, bool check_ta_flush, + bool is_reset, uint32_t tp_num) +{ + int ret = 0; + + if (check_ta_flush) + return ret; + + if (check_feature_enable && !(dev->caps.feature & UDMA_CAP_FEATURE_UE_RX_CLOSE)) + return ret; + + mutex_lock(&dev->disable_ue_rx_mutex); + if (tp_num) + dev->disable_ue_rx_count -= tp_num; + else + dev->disable_ue_rx_count--; + if (dev->disable_ue_rx_count == 0 && !is_reset) { + ret = ubase_activate_dev(dev->comdev.adev); + if (ret) + dev_err(dev->dev, "failed to open ue rx, ret = %d.\n", ret); + } + mutex_unlock(&dev->disable_ue_rx_mutex); + + return ret; +} + +module_param(debug_switch, bool, 0444); +MODULE_PARM_DESC(debug_switch, "set debug print ON, default: true"); diff --git a/drivers/ub/urma/hw/udma/udma_cmd.h b/drivers/ub/urma/hw/udma/udma_cmd.h new file mode 100644 index 0000000000000000000000000000000000000000..24f6d65bf1adec380c3ff835acd2a5d155d9ceae --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_cmd.h @@ -0,0 +1,305 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_CMD_H__ +#define __UDMA_CMD_H__ + +#include +#include +#include +#include "udma_dev.h" + +extern bool debug_switch; + +#define UDMA_MAILBOX_SIZE 4096 + +#define SPEED_200G 200000 +#define SPEED_400G 400000 +#define SPEED_100G 100000 +#define SPEED_50G 50000 +#define SPEED_25G 25000 + +#define UDMA_CTRLQ_SEID_NUM 64 + +struct udma_ctrlq_eid_info { + uint32_t eid_idx; + union ubcore_eid eid; + uint32_t upi; +} __packed; + +struct udma_ctrlq_eid_in_query { + uint32_t cmd : 8; + uint32_t rsv : 24; +}; + +struct udma_ctrlq_eid_out_query { + uint32_t seid_num : 8; + uint32_t rsv : 24; + struct udma_ctrlq_eid_info eids[UDMA_CTRLQ_SEID_NUM]; +} __packed; + +struct udma_ctrlq_eid_out_update { + struct udma_ctrlq_eid_info eid_info; + uint32_t op_type : 4; + uint32_t rsv : 28; +} __packed; + +enum udma_ctrlq_eid_update_op { + UDMA_CTRLQ_EID_ADD = 0, + UDMA_CTRLQ_EID_DEL, +}; + +enum udma_ctrlq_eid_guid_update_op { + UDMA_CTRLQ_EID_GUID_ADD = 0, + UDMA_CTRLQ_EID_GUID_DEL, +}; + +struct udma_ctrlq_ue_eid_guid_out { + struct udma_ctrlq_eid_info eid_info; + uint32_t op_type : 4; + uint32_t rsv : 28; + uint32_t ue_id; + guid_t ue_guid; +} __packed; + +enum udma_ctrlq_dev_mgmt_opcode { + UDMA_CTRLQ_GET_SEID_INFO = 0x1, + UDMA_CTRLQ_UPDATE_SEID_INFO = 0x2, + UDMA_CTRLQ_OPC_UPDATE_UE_SEID_GUID = 0x3, + UDMA_CTRLQ_GET_DEV_RESOURCE_COUNT = 0x11, + UDMA_CTRLQ_GET_DEV_RESOURCE_RATIO = 0x12, + UDMA_CTRLQ_NOTIFY_DEV_RESOURCE_RATIO = 0x13, +}; + +enum udma_cmd_opcode_type { + UDMA_CMD_QUERY_UE_RES = 0x0002, + UDMA_CMD_QUERY_UE_INDEX = 0x241d, + UDMA_CMD_CFG_CONG_PARAM = 0x3003, + UDMA_CMD_CHANGE_ACTIVE_PORT = 0x3102, + UDMA_CMD_DEBUGFS_TP_INFO = 0x3110, + UDMA_CMD_DEBUGFS_TA_INFO = 0x4210, + UDMA_CMD_GET_CQE_AUX_INFO = 0x4213, + UDMA_CMD_GET_AE_AUX_INFO = 0x4214, + UDMA_CMD_QUERY_PORT_INFO = 0x6200, + UDMA_CMD_WQEBB_VA_INFO = 0xa01f, +}; + +struct udma_cmd { + uint32_t opcode; + void *in_buf; + uint32_t in_len; + void *out_buf; + uint32_t out_len; +}; + +enum { + /* JFS CMDS */ + UDMA_CMD_WRIET_JFS_CONTEXT_VA = 0x00, + UDMA_CMD_READ_JFS_CONTEXT_VA = 0x01, + UDMA_CMD_DESTROY_JFS_CONTEXT_VA = 0x02, + UDMA_CMD_CREATE_JFS_CONTEXT = 0x04, + UDMA_CMD_MODIFY_JFS_CONTEXT = 0x05, + UDMA_CMD_QUERY_JFS_CONTEXT = 0x06, + UDMA_CMD_DESTROY_JFS_CONTEXT = 0x07, + + /* RC CMDS */ + UDMA_CMD_WRITE_RC_CONTEXT_VA = 0x10, + UDMA_CMD_READ_RC_CONTEXT_VA = 0x11, + UDMA_CMD_DESTROY_RC_CONTEXT_VA = 0x12, + UDMA_CMD_CREATE_RC_CONTEXT = 0x14, + UDMA_CMD_MODIFY_RC_CONTEXT = 0x15, + UDMA_CMD_QUERY_RC_CONTEXT = 0X16, + UDMA_CMD_DESTROY_RC_CONTEXT = 0x17, + + /* JFC CMDS */ + UDMA_CMD_WRITE_JFC_CONTEXT_VA = 0x20, + UDMA_CMD_READ_JFC_CONTEXT_VA = 0x21, + UDMA_CMD_DESTROY_JFC_CONTEXT_VA = 0x22, + UDMA_CMD_CREATE_JFC_CONTEXT = 0x24, + UDMA_CMD_MODIFY_JFC_CONTEXT = 0x25, + UDMA_CMD_QUERY_JFC_CONTEXT = 0x26, + UDMA_CMD_DESTROY_JFC_CONTEXT = 0x27, + + /* CEQ CMDS */ + UDMA_CMD_CREATE_CEQ_CONTEXT = 0x44, + UDMA_CMD_MODIFY_CEQ_CONTEXT = 0x45, + UDMA_CMD_QUERY_CEQ_CONTEXT = 0x46, + UDMA_CMD_DESTROY_CEQ_CONTEXT = 0x47, + + /* JFR CMDS */ + UDMA_CMD_WRITE_JFR_CONTEXT_VA = 0x50, + UDMA_CMD_READ_JFR_CONTEXT_VA = 0x51, + UDMA_CMD_DESTROY_JFR_CONTEXT_VA = 0x52, + UDMA_CMD_CREATE_JFR_CONTEXT = 0x54, + UDMA_CMD_MODIFY_JFR_CONTEXT = 0x55, + UDMA_CMD_QUERY_JFR_CONTEXT = 0x56, + UDMA_CMD_DESTROY_JFR_CONTEXT = 0x57, + + /* JETTY CMDS */ + UDMA_CMD_WRITE_JETTY_GROUP_CONTEXT_VA = 0x60, + UDMA_CMD_READ_JETTY_GROUP_CONTEXT_VA = 0x61, + UDMA_CMD_DESTROY_JETTY_GROUP_CONTEXT_VA = 0x62, + UDMA_CMD_CREATE_JETTY_GROUP_CONTEXT = 0x64, + UDMA_CMD_MODIFY_JETTY_GROUP_CONTEXT = 0x65, + UDMA_CMD_QUERY_JETTY_GROUP_CONTEXT = 0x66, + UDMA_CMD_DESTROY_JETTY_GROUP_CONTEXT = 0x67, + + /* TP CMDS */ + UDMA_CMD_QUERY_TP_CONTEXT = 0x86, + + /* SEID_UPI CMDS */ + UDMA_CMD_READ_SEID_UPI = 0xb5, +}; + +struct udma_mbx_op_match { + uint32_t op; + bool ignore_ret; + uint32_t entry_size; +}; + +struct cap_info { + uint16_t ar_en : 1; + uint16_t jfc_per_wr : 1; + uint16_t stride_up : 1; + uint16_t load_store_op : 1; + uint16_t jfc_inline : 1; + uint16_t non_pin : 1; + uint16_t selective_retrans : 1; + uint16_t rsvd : 9; + uint16_t rsvd1; +}; + +struct udma_cmd_ue_resource { + /* BD0 */ + uint16_t jfs_num_shift : 4; + uint16_t jfr_num_shift : 4; + uint16_t jfc_num_shift : 4; + uint16_t jetty_num_shift : 4; + + uint16_t jetty_grp_num; + + uint16_t jfs_depth_shift : 4; + uint16_t jfr_depth_shift : 4; + uint16_t jfc_depth_shift : 4; + uint16_t cqe_size_shift : 4; + + uint16_t jfs_sge : 5; + uint16_t jfr_sge : 5; + uint16_t jfs_rsge : 6; + + uint16_t max_jfs_inline_sz; + uint16_t max_jfc_inline_sz; + uint32_t cap_info; + + uint16_t trans_mode : 5; + uint16_t ue_num : 8; + uint16_t virtualization : 1; + uint16_t dcqcn_sw_en : 1; + uint16_t rsvd0 : 1; + + uint16_t ue_cnt; + uint8_t ue_id; + uint8_t default_cong_alg; + uint8_t cons_ctrl_alg; + uint8_t cc_priority_cnt; + + /* BD1 */ + uint16_t src_addr_tbl_sz; + uint16_t src_addr_tbl_num; + uint16_t dest_addr_tbl_sz; + uint16_t dest_addr_tbl_num; + uint16_t seid_upi_tbl_sz; + uint16_t seid_upi_tbl_num; + uint16_t tpm_tbl_sz; + uint16_t tpm_tbl_num; + uint32_t tp_range; + uint8_t port_num; + uint8_t port_id; + uint8_t rsvd1[2]; + uint16_t rc_queue_num; + uint16_t rc_depth; + uint8_t rc_entry; + uint8_t rsvd2[3]; + + /* BD2 */ + uint16_t well_known_jetty_start; + uint16_t well_known_jetty_num; + uint16_t ccu_jetty_start; + uint16_t ccu_jetty_num; + uint16_t drv_jetty_start; + uint16_t drv_jetty_num; + uint16_t cache_lock_jetty_start; + uint16_t cache_lock_jetty_num; + uint16_t normal_jetty_start; + uint16_t normal_jetty_num; + uint16_t standard_jetty_start; + uint16_t standard_jetty_num; + uint32_t rsvd3[2]; + + /* BD3 */ + uint32_t max_write_size; + uint32_t max_read_size; + uint32_t max_cas_size; + uint32_t max_fetch_and_add_size; + uint32_t atomic_feat; + uint32_t rsvd4[3]; +}; + +struct udma_cmd_port_info { + uint32_t speed; + uint8_t rsv[10]; + uint8_t lanes; + uint8_t rsv2[9]; +}; + +struct udma_cmd_wqebb_va { + uint64_t va_start; + uint64_t va_size; + uint32_t die_num; + uint32_t ue_num; +}; + +struct udma_cmd_query_cqe_aux_info { + uint32_t status : 8; + uint32_t is_client : 1; + uint32_t rsvd : 23; + uint32_t cqe_aux_info[MAX_CQE_AUX_INFO_TYPE_NUM]; +}; + +struct udma_cmd_query_ae_aux_info { + uint32_t event_type : 8; + uint32_t sub_type : 8; + uint32_t rsvd : 16; + uint32_t ae_aux_info[MAX_AE_AUX_INFO_TYPE_NUM]; +}; + +static inline void udma_fill_buf(struct ubase_cmd_buf *buf, u16 opcode, + bool is_read, u32 data_size, void *data) +{ + buf->opcode = opcode; + buf->is_read = is_read; + buf->data_size = data_size; + buf->data = data; +} + +int udma_cmd_init(struct udma_dev *udma_dev); +void udma_cmd_cleanup(struct udma_dev *udma_dev); +struct ubase_cmd_mailbox *udma_alloc_cmd_mailbox(struct udma_dev *dev); +void udma_free_cmd_mailbox(struct udma_dev *dev, + struct ubase_cmd_mailbox *mailbox); +int udma_post_mbox(struct udma_dev *dev, struct ubase_cmd_mailbox *mailbox, + struct ubase_mbx_attr *attr); +int udma_cmd_query_hw_resource(struct udma_dev *udma_dev, void *out_addr); +int udma_config_ctx_buf_to_hw(struct udma_dev *udma_dev, + struct udma_buf *ctx_buf, + struct ubase_mbx_attr *attr); +int post_mailbox_update_ctx(struct udma_dev *udma_dev, void *ctx, uint32_t size, + struct ubase_mbx_attr *attr); +struct ubase_cmd_mailbox *udma_mailbox_query_ctx(struct udma_dev *udma_dev, + struct ubase_mbx_attr *attr); +int udma_close_ue_rx(struct udma_dev *dev, bool check_feature_enable, bool check_ta_flush, + bool is_reset, uint32_t tp_num); +int udma_open_ue_rx(struct udma_dev *dev, bool check_feature_enable, bool check_ta_flush, + bool is_reset, uint32_t tp_num); + +#endif /* __UDMA_CMD_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_common.c b/drivers/ub/urma/hw/udma/udma_common.c new file mode 100644 index 0000000000000000000000000000000000000000..07d57a5ce96b2bd87d3ab2644b7daffc5ac4b4bb --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_common.c @@ -0,0 +1,872 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "udma_dev.h" +#include "udma_cmd.h" +#include "udma_common.h" + +static int udma_verify_input(struct udma_umem_param *param) +{ + struct udma_dev *udma_dev = to_udma_dev(param->ub_dev); + + if (((param->va + param->len) < param->va) || + PAGE_ALIGN(param->va + param->len) < (param->va + param->len)) { + dev_err(udma_dev->dev, "invalid pin_page param, len=%llu.\n", + param->len); + return -EINVAL; + } + return 0; +} + +static void udma_fill_umem(struct ubcore_umem *umem, struct udma_umem_param *param) +{ + umem->ub_dev = param->ub_dev; + umem->va = param->va; + umem->length = param->len; + umem->flag = param->flag; +} + +static struct scatterlist *udma_sg_set_page(struct scatterlist *sg_start, + int pinned, struct page **page_list) +{ + struct scatterlist *sg; + int i; + + for_each_sg(sg_start, sg, pinned, i) + sg_set_page(sg, page_list[i], PAGE_SIZE, 0); + + return sg; +} + +static int udma_pin_pages(uint64_t cur_base, uint64_t npages, + uint32_t gup_flags, struct page **page_list) +{ + return pin_user_pages_fast(cur_base, min_t(unsigned long, (unsigned long)npages, + PAGE_SIZE / sizeof(struct page *)), + gup_flags | FOLL_LONGTERM, page_list); +} + +static uint64_t udma_pin_all_pages(struct udma_dev *udma_dev, struct ubcore_umem *umem, + uint64_t npages, uint32_t gup_flags, + struct page **page_list) +{ + struct scatterlist *sg_list_start = umem->sg_head.sgl; + uint64_t cur_base = umem->va & PAGE_MASK; + uint64_t page_count = npages; + int pinned; + + while (page_count != 0) { + cond_resched(); + pinned = udma_pin_pages(cur_base, page_count, gup_flags, page_list); + if (pinned <= 0) { + dev_err(udma_dev->dev, "failed to pin_user_pages_fast, page_count: %llu, pinned: %d.\n", + page_count, pinned); + return npages - page_count; + } + cur_base += (uint64_t)pinned * PAGE_SIZE; + page_count -= (uint64_t)pinned; + sg_list_start = udma_sg_set_page(sg_list_start, pinned, page_list); + } + return npages; +} + +static uint64_t udma_k_pin_pages(struct udma_dev *dev, struct ubcore_umem *umem, + uint64_t npages) +{ + struct scatterlist *sg_cur = umem->sg_head.sgl; + uint64_t cur_base = umem->va & PAGE_MASK; + struct page *pg; + uint64_t pinned; + + for (pinned = 0; pinned < npages; pinned++) { + if (is_vmalloc_addr((void *)(uintptr_t)cur_base)) + pg = vmalloc_to_page((void *)(uintptr_t)cur_base); + else + pg = kmap_to_page((void *)(uintptr_t)cur_base); + if (!pg) { + dev_err(dev->dev, "vmalloc or kmap to page failed.\n"); + break; + } + get_page(pg); + + cur_base += PAGE_SIZE; + + sg_set_page(sg_cur, pg, PAGE_SIZE, 0); + sg_cur = sg_next(sg_cur); + } + + return pinned; +} + +static void udma_unpin_pages(struct ubcore_umem *umem, uint64_t nents, bool is_kernel) +{ + struct scatterlist *sg; + uint32_t i; + + for_each_sg(umem->sg_head.sgl, sg, nents, i) { + struct page *page = sg_page(sg); + + if (is_kernel) + put_page(page); + else + unpin_user_page(page); + } +} + +static struct ubcore_umem *udma_get_target_umem(struct udma_umem_param *param, + struct page **page_list) +{ + struct udma_dev *udma_dev = to_udma_dev(param->ub_dev); + struct ubcore_umem *umem; + uint32_t gup_flags; + uint64_t npages; + uint64_t pinned; + int ret = 0; + + umem = kzalloc(sizeof(*umem), GFP_KERNEL); + if (umem == 0) { + ret = -ENOMEM; + goto out; + } + + udma_fill_umem(umem, param); + + npages = udma_cal_npages(umem->va, umem->length); + if (npages == 0 || npages > UINT_MAX) { + dev_err(udma_dev->dev, + "Invalid npages %llu in getting target umem process.\n", npages); + ret = -EINVAL; + goto umem_kfree; + } + + ret = sg_alloc_table(&umem->sg_head, (unsigned int)npages, GFP_KERNEL); + if (ret) + goto umem_kfree; + + if (param->is_kernel) { + pinned = udma_k_pin_pages(udma_dev, umem, npages); + } else { + gup_flags = (param->flag.bs.writable == 1) ? FOLL_WRITE : 0; + pinned = udma_pin_all_pages(udma_dev, umem, npages, gup_flags, page_list); + } + if (pinned != npages) { + ret = -ENOMEM; + goto umem_release; + } + + goto out; + +umem_release: + udma_unpin_pages(umem, pinned, param->is_kernel); + sg_free_table(&umem->sg_head); +umem_kfree: + kfree(umem); +out: + return ret != 0 ? ERR_PTR(ret) : umem; +} + +struct ubcore_umem *udma_umem_get(struct udma_umem_param *param) +{ + struct ubcore_umem *umem; + struct page **page_list; + int ret; + + ret = udma_verify_input(param); + if (ret < 0) + return ERR_PTR(ret); + + page_list = (struct page **) __get_free_page(GFP_KERNEL); + if (page_list == 0) + return ERR_PTR(-ENOMEM); + + umem = udma_get_target_umem(param, page_list); + + free_page((uintptr_t)page_list); + + return umem; +} + +int pin_queue_addr(struct udma_dev *dev, uint64_t addr, uint32_t len, + struct udma_buf *buf) +{ + struct ubcore_device *ub_dev = &dev->ub_dev; + struct udma_umem_param param; + + param.ub_dev = ub_dev; + param.va = addr; + param.len = len; + param.flag.bs.writable = 1; + param.flag.bs.non_pin = 0; + param.is_kernel = false; + + buf->umem = udma_umem_get(¶m); + if (IS_ERR(buf->umem)) { + dev_err(dev->dev, "failed to pin queue addr.\n"); + return PTR_ERR(buf->umem); + } + + buf->addr = addr; + + return 0; +} + +void unpin_queue_addr(struct ubcore_umem *umem) +{ + udma_umem_release(umem, false); +} + +void udma_umem_release(struct ubcore_umem *umem, bool is_kernel) +{ + if (IS_ERR_OR_NULL(umem)) + return; + + udma_unpin_pages(umem, umem->sg_head.nents, is_kernel); + sg_free_table(&umem->sg_head); + kfree(umem); +} + +int udma_id_alloc_auto_grow(struct udma_dev *udma_dev, struct udma_ida *ida_table, + uint32_t *idx) +{ + int id; + + spin_lock(&ida_table->lock); + id = ida_alloc_range(&ida_table->ida, ida_table->next, ida_table->max, + GFP_ATOMIC); + if (id < 0) { + id = ida_alloc_range(&ida_table->ida, ida_table->min, ida_table->max, + GFP_ATOMIC); + if (id < 0) { + dev_err(udma_dev->dev, "failed to alloc id, ret = %d.\n", id); + spin_unlock(&ida_table->lock); + return id; + } + } + + ida_table->next = (uint32_t)id + 1 > ida_table->max ? + ida_table->min : (uint32_t)id + 1; + + *idx = (uint32_t)id; + spin_unlock(&ida_table->lock); + + return 0; +} + +int udma_id_alloc(struct udma_dev *udma_dev, struct udma_ida *ida_table, + uint32_t *idx) +{ + int id; + + id = ida_alloc_range(&ida_table->ida, ida_table->min, ida_table->max, + GFP_ATOMIC); + if (id < 0) { + dev_err(udma_dev->dev, "failed to alloc id, ret = %d.\n", id); + return id; + } + + *idx = (uint32_t)id; + + return 0; +} + +int udma_specify_adv_id(struct udma_dev *udma_dev, struct udma_group_bitmap *bitmap_table, + uint32_t user_id) +{ + uint32_t id_bit_idx = (user_id - bitmap_table->min); + uint32_t bit_idx = id_bit_idx % NUM_JETTY_PER_GROUP; + uint32_t block = id_bit_idx / NUM_JETTY_PER_GROUP; + uint32_t *bit = bitmap_table->bit; + + spin_lock(&bitmap_table->lock); + if ((bit[block] & (1U << bit_idx)) == 0) { + dev_err(udma_dev->dev, + "user specify id %u been used.\n", user_id); + spin_unlock(&bitmap_table->lock); + return -ENOMEM; + } + + bit[block] &= ~(1U << bit_idx); + spin_unlock(&bitmap_table->lock); + + return 0; +} + +static int udma_adv_jetty_id_alloc(struct udma_dev *udma_dev, uint32_t *bit, + uint32_t next_bit, uint32_t start_idx, + struct udma_group_bitmap *bitmap_table) +{ + uint32_t bit_idx; + + bit_idx = find_next_bit((unsigned long *)bit, NUM_JETTY_PER_GROUP, next_bit); + if (bit_idx == NUM_JETTY_PER_GROUP) { + dev_err(udma_dev->dev, + "jid is larger than n_bits, bit=0x%x.\n", *bit); + return -ENOMEM; + } + + start_idx += bit_idx; + if (start_idx >= bitmap_table->n_bits) { + dev_err(udma_dev->dev, + "jid is larger than n_bits, id=%u, n_bits=%u.\n", + start_idx, bitmap_table->n_bits); + return -ENOMEM; + } + + *bit &= ~(1U << bit_idx); + return start_idx + bitmap_table->min; +} + +int udma_adv_id_alloc(struct udma_dev *udma_dev, struct udma_group_bitmap *bitmap_table, + uint32_t *start_idx, bool is_grp, uint32_t next) +{ + uint32_t next_block = (next - bitmap_table->min) / NUM_JETTY_PER_GROUP; + uint32_t next_bit = (next - bitmap_table->min) % NUM_JETTY_PER_GROUP; + uint32_t bitmap_cnt = bitmap_table->bitmap_cnt; + uint32_t *bit = bitmap_table->bit; + uint32_t i; + int ret; + + spin_lock(&bitmap_table->lock); + + for (i = next_block; + (i < bitmap_cnt && bit[i] == 0) || + (i == next_block && + ((bit[i] & GENMASK(NUM_JETTY_PER_GROUP - 1, next_bit)) == 0)); ++i) + ; + + if (i == bitmap_cnt) { + dev_err(udma_dev->dev, + "all bitmaps have been used, bitmap_cnt = %u.\n", + bitmap_cnt); + spin_unlock(&bitmap_table->lock); + return -ENOMEM; + } + + if (!is_grp) { + ret = udma_adv_jetty_id_alloc(udma_dev, bit + i, next_bit, + i * NUM_JETTY_PER_GROUP, bitmap_table); + + spin_unlock(&bitmap_table->lock); + if (ret >= 0) { + *start_idx = (uint32_t)ret; + return 0; + } + return ret; + } + + for (; i < bitmap_cnt && ~bit[i] != 0; ++i) + ; + if (i == bitmap_cnt || + (i + 1) * NUM_JETTY_PER_GROUP > bitmap_table->n_bits) { + dev_err(udma_dev->dev, + "no completely bitmap for Jetty group.\n"); + spin_unlock(&bitmap_table->lock); + return -ENOMEM; + } + + bit[i] = 0; + *start_idx = i * NUM_JETTY_PER_GROUP + bitmap_table->min; + + spin_unlock(&bitmap_table->lock); + + return 0; +} + +void udma_adv_id_free(struct udma_group_bitmap *bitmap_table, uint32_t start_idx, + bool is_grp) +{ + uint32_t bitmap_num; + uint32_t bit_num; + + start_idx -= bitmap_table->min; + spin_lock(&bitmap_table->lock); + + bitmap_num = start_idx / NUM_JETTY_PER_GROUP; + if (bitmap_num >= bitmap_table->bitmap_cnt) { + spin_unlock(&bitmap_table->lock); + return; + } + + if (is_grp) { + bitmap_table->bit[bitmap_num] = ~0U; + } else { + bit_num = start_idx % NUM_JETTY_PER_GROUP; + bitmap_table->bit[bitmap_num] |= (1U << bit_num); + } + + spin_unlock(&bitmap_table->lock); +} + +static void udma_init_ida_table(struct udma_ida *ida_table, uint32_t max, uint32_t min) +{ + ida_init(&ida_table->ida); + spin_lock_init(&ida_table->lock); + ida_table->max = max; + ida_table->min = min; + ida_table->next = min; +} + +void udma_init_udma_table(struct udma_table *table, uint32_t max, uint32_t min) +{ + udma_init_ida_table(&table->ida_table, max, min); + xa_init(&table->xa); +} + +void udma_init_udma_table_mutex(struct xarray *table, struct mutex *udma_mutex) +{ + xa_init(table); + mutex_init(udma_mutex); +} + +void udma_destroy_npu_cb_table(struct udma_dev *dev) +{ + struct udma_ctrlq_event_nb *nb = NULL; + unsigned long index = 0; + + mutex_lock(&dev->npu_nb_mutex); + if (!xa_empty(&dev->npu_nb_table)) { + xa_for_each(&dev->npu_nb_table, index, nb) { + ubase_ctrlq_unregister_crq_event(dev->comdev.adev, + UBASE_CTRLQ_SER_TYPE_DEV_REGISTER, + nb->opcode); + __xa_erase(&dev->npu_nb_table, index); + kfree(nb); + nb = NULL; + } + } + + mutex_unlock(&dev->npu_nb_mutex); + xa_destroy(&dev->npu_nb_table); + mutex_destroy(&dev->npu_nb_mutex); +} + +void udma_destroy_udma_table(struct udma_dev *dev, struct udma_table *table, + const char *table_name) +{ + if (!ida_is_empty(&table->ida_table.ida)) + dev_err(dev->dev, "IDA not empty in clean up %s table.\n", + table_name); + ida_destroy(&table->ida_table.ida); + + if (!xa_empty(&table->xa)) + dev_err(dev->dev, "%s not empty.\n", table_name); + xa_destroy(&table->xa); +} + +static void udma_clear_eid_table(struct udma_dev *udma_dev) +{ + struct udma_ctrlq_eid_info *eid_entry = NULL; + unsigned long index = 0; + eid_t ummu_eid = 0; + guid_t guid = {}; + + if (!xa_empty(&udma_dev->eid_table)) { + xa_for_each(&udma_dev->eid_table, index, eid_entry) { + xa_erase(&udma_dev->eid_table, index); + if (!udma_dev->is_ue) { + (void)memcpy(&ummu_eid, eid_entry->eid.raw, sizeof(ummu_eid)); + ummu_core_del_eid(&guid, ummu_eid, EID_NONE); + } + kfree(eid_entry); + eid_entry = NULL; + } + } +} + +void udma_destroy_eid_table(struct udma_dev *udma_dev) +{ + udma_clear_eid_table(udma_dev); + xa_destroy(&udma_dev->eid_table); + mutex_destroy(&udma_dev->eid_mutex); +} + +void udma_dfx_store_id(struct udma_dev *udma_dev, struct udma_dfx_entity *entity, + uint32_t id, const char *name) +{ + uint32_t *entry; + int ret; + + entry = (uint32_t *)xa_load(&entity->table, id); + if (entry) { + dev_warn(udma_dev->dev, "%s(%u) already exists in DFX.\n", name, id); + return; + } + + entry = kzalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return; + + *entry = id; + + write_lock(&entity->rwlock); + ret = xa_err(xa_store(&entity->table, id, entry, GFP_KERNEL)); + if (ret) { + write_unlock(&entity->rwlock); + dev_err(udma_dev->dev, "store %s to table failed in DFX.\n", name); + kfree(entry); + return; + } + + ++entity->cnt; + write_unlock(&entity->rwlock); +} + +void udma_dfx_delete_id(struct udma_dev *udma_dev, struct udma_dfx_entity *entity, + uint32_t id) +{ + void *entry; + + write_lock(&entity->rwlock); + entry = xa_load(&entity->table, id); + if (!entry) { + write_unlock(&entity->rwlock); + return; + } + + xa_erase(&entity->table, id); + kfree(entry); + --entity->cnt; + write_unlock(&entity->rwlock); +} + +static struct ubcore_umem *udma_pin_k_addr(struct ubcore_device *ub_dev, uint64_t va, + uint64_t len) +{ + struct udma_umem_param param; + + param.ub_dev = ub_dev; + param.va = va; + param.len = len; + param.flag.bs.writable = true; + param.flag.bs.non_pin = 0; + param.is_kernel = true; + + return udma_umem_get(¶m); +} + +static void udma_unpin_k_addr(struct ubcore_umem *umem) +{ + udma_umem_release(umem, true); +} + +int udma_alloc_normal_buf(struct udma_dev *udma_dev, size_t memory_size, + struct udma_buf *buf) +{ + size_t aligned_memory_size; + int ret; + + aligned_memory_size = memory_size + UDMA_HW_PAGE_SIZE - 1; + buf->aligned_va = vmalloc(aligned_memory_size); + if (!buf->aligned_va) { + dev_err(udma_dev->dev, + "failed to vmalloc kernel buf, size = %lu.", + aligned_memory_size); + return -ENOMEM; + } + + memset(buf->aligned_va, 0, aligned_memory_size); + buf->umem = udma_pin_k_addr(&udma_dev->ub_dev, (uint64_t)buf->aligned_va, + aligned_memory_size); + if (IS_ERR(buf->umem)) { + ret = PTR_ERR(buf->umem); + vfree(buf->aligned_va); + dev_err(udma_dev->dev, "pin kernel buf failed, ret = %d.\n", ret); + return ret; + } + + buf->addr = ((uint64_t)buf->aligned_va + UDMA_HW_PAGE_SIZE - 1) & + ~(UDMA_HW_PAGE_SIZE - 1); + buf->kva = (void *)(uintptr_t)buf->addr; + + return 0; +} + +void udma_free_normal_buf(struct udma_dev *udma_dev, size_t memory_size, + struct udma_buf *buf) +{ + udma_unpin_k_addr(buf->umem); + vfree(buf->aligned_va); + buf->aligned_va = NULL; + buf->kva = NULL; + buf->addr = 0; +} + +static struct udma_hugepage_priv * +udma_alloc_hugepage_priv(struct udma_dev *dev, uint32_t len) +{ + struct udma_hugepage_priv *priv; + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return NULL; + + priv->va_len = ALIGN(len, UDMA_HUGEPAGE_SIZE); + if (priv->va_len >> UDMA_HUGEPAGE_SHIFT > dev->total_hugepage_num) { + dev_err(dev->dev, "insufficient resources for mmap.\n"); + goto err_vmalloc_huge; + } + + priv->left_va_len = priv->va_len; + priv->va_base = vmalloc_huge(priv->va_len, GFP_KERNEL); + if (!priv->va_base) { + dev_err(dev->dev, "failed to vmalloc_huge, size=%u.", priv->va_len); + goto err_vmalloc_huge; + } + memset(priv->va_base, 0, priv->va_len); + + priv->umem = udma_pin_k_addr(&dev->ub_dev, (uint64_t)priv->va_base, priv->va_len); + if (IS_ERR(priv->umem)) { + dev_err(dev->dev, "pin kernel buf failed.\n"); + goto err_pin; + } + + refcount_set(&priv->refcnt, 1); + list_add(&priv->list, &dev->hugepage_list); + dev->total_hugepage_num -= priv->va_len >> UDMA_HUGEPAGE_SHIFT; + + if (dfx_switch) + dev_info_ratelimited(dev->dev, "map_hugepage, 2m_page_num=%u.\n", + priv->va_len >> UDMA_HUGEPAGE_SHIFT); + return priv; + +err_pin: + vfree(priv->va_base); +err_vmalloc_huge: + kfree(priv); + + return NULL; +} + +static struct udma_hugepage * +udma_alloc_hugepage(struct udma_dev *dev, uint32_t len) +{ + struct udma_hugepage_priv *priv = NULL; + struct udma_hugepage *hugepage; + bool b_reuse = false; + + hugepage = kzalloc(sizeof(*hugepage), GFP_KERNEL); + if (!hugepage) + return NULL; + + mutex_lock(&dev->hugepage_lock); + if (!list_empty(&dev->hugepage_list)) { + priv = list_first_entry(&dev->hugepage_list, struct udma_hugepage_priv, list); + b_reuse = len <= priv->left_va_len; + } + + if (b_reuse) { + refcount_inc(&priv->refcnt); + } else { + priv = udma_alloc_hugepage_priv(dev, len); + if (!priv) { + mutex_unlock(&dev->hugepage_lock); + kfree(hugepage); + return NULL; + } + } + + hugepage->va_start = priv->va_base + priv->left_va_offset; + hugepage->va_len = len; + hugepage->priv = priv; + priv->left_va_offset += len; + priv->left_va_len -= len; + mutex_unlock(&dev->hugepage_lock); + + if (dfx_switch) + dev_info_ratelimited(dev->dev, "occupy_hugepage, 4k_page_num=%u.\n", + hugepage->va_len >> UDMA_HW_PAGE_SHIFT); + return hugepage; +} + +static void udma_free_hugepage(struct udma_dev *dev, struct udma_hugepage *hugepage) +{ + struct udma_hugepage_priv *priv = hugepage->priv; + + if (dfx_switch) + dev_info_ratelimited(dev->dev, "return_hugepage, 4k_page_num=%u.\n", + hugepage->va_len >> UDMA_HW_PAGE_SHIFT); + mutex_lock(&dev->hugepage_lock); + if (refcount_dec_and_test(&priv->refcnt)) { + if (dfx_switch) + dev_info_ratelimited(dev->dev, "unmap_hugepage, 2m_page_num=%u.\n", + priv->va_len >> UDMA_HUGEPAGE_SHIFT); + list_del(&priv->list); + dev->total_hugepage_num += priv->va_len >> UDMA_HUGEPAGE_SHIFT; + + udma_unpin_k_addr(priv->umem); + vfree(priv->va_base); + kfree(priv); + } else { + memset(hugepage->va_start, 0, hugepage->va_len); + } + mutex_unlock(&dev->hugepage_lock); + kfree(hugepage); +} + +int udma_k_alloc_buf(struct udma_dev *dev, struct udma_buf *buf) +{ + uint32_t size = buf->entry_size * buf->entry_cnt; + uint32_t hugepage_size; + int ret = 0; + + if (ubase_adev_prealloc_supported(dev->comdev.adev)) { + hugepage_size = ALIGN(size, UDMA_HW_PAGE_SIZE); + buf->hugepage = udma_alloc_hugepage(dev, hugepage_size); + if (buf->hugepage) { + buf->kva = buf->hugepage->va_start; + buf->addr = (uint64_t)buf->kva; + buf->is_hugepage = true; + } else { + dev_warn(dev->dev, + "failed to alloc hugepage buf, switch to alloc normal buf."); + ret = udma_alloc_normal_buf(dev, size, buf); + } + } else { + ret = udma_alloc_normal_buf(dev, size, buf); + } + + return ret; +} + +void udma_k_free_buf(struct udma_dev *dev, struct udma_buf *buf) +{ + uint32_t size = buf->entry_cnt * buf->entry_size; + + if (buf->is_hugepage) + udma_free_hugepage(dev, buf->hugepage); + else + udma_free_normal_buf(dev, size, buf); +} + +void *udma_alloc_iova(struct udma_dev *udma_dev, size_t memory_size, dma_addr_t *addr) +{ + struct iova_slot *slot; + uint32_t npage; + size_t sizep; + int ret; + + slot = dma_alloc_iova(udma_dev->dev, memory_size, 0, addr, &sizep); + if (IS_ERR_OR_NULL(slot)) { + dev_err(udma_dev->dev, + "failed to dma alloc iova, size = %lu, ret = %ld.\n", + memory_size, PTR_ERR(slot)); + return NULL; + } + + npage = sizep >> PAGE_SHIFT; + ret = ummu_fill_pages(slot, *addr, npage); + if (ret) { + dev_err(udma_dev->dev, + "ummu fill pages failed, npage = %u, ret = %d", npage, ret); + dma_free_iova(slot); + return NULL; + } + + return (void *)slot; +} + +void udma_free_iova(struct udma_dev *udma_dev, size_t memory_size, void *kva_or_slot, + dma_addr_t addr) +{ + size_t aligned_memory_size; + struct iova_slot *slot; + uint32_t npage; + int ret; + + aligned_memory_size = PAGE_ALIGN(memory_size); + npage = aligned_memory_size >> PAGE_SHIFT; + slot = (struct iova_slot *)kva_or_slot; + ret = ummu_drain_pages(slot, addr, npage); + if (ret) + dev_err(udma_dev->dev, + "ummu drain pages failed, npage = %u, ret = %d.\n", + npage, ret); + + dma_free_iova(slot); +} + +int udma_query_ue_idx(struct ubcore_device *ubcore_dev, struct ubcore_devid *devid, + uint16_t *ue_idx) +{ + struct udma_dev *dev = to_udma_dev(ubcore_dev); + struct udma_ue_index_cmd cmd = {}; + struct ubase_cmd_buf out; + struct ubase_cmd_buf in; + int ret; + + if (!devid) { + dev_err(dev->dev, "failed to query ue idx, devid is NULL.\n"); + return -EINVAL; + } + + (void)memcpy(cmd.guid, devid->raw, sizeof(devid->raw)); + + udma_fill_buf(&in, UDMA_CMD_QUERY_UE_INDEX, true, sizeof(cmd), &cmd); + udma_fill_buf(&out, UDMA_CMD_QUERY_UE_INDEX, true, sizeof(cmd), &cmd); + + ret = ubase_cmd_send_inout(dev->comdev.adev, &in, &out); + if (ret) { + dev_err(dev->dev, "failed to query ue idx, ret = %d.\n", ret); + return ret; + } + *ue_idx = cmd.ue_idx; + + return 0; +} + +void udma_dfx_ctx_print(struct udma_dev *udev, const char *name, uint32_t id, uint32_t len, + uint32_t *ctx) +{ + uint32_t i; + + pr_info("*************udma%u %s(%u) CONTEXT INFO *************\n", + udev->adev_id, name, id); + + for (i = 0; i < len; ++i) + pr_info("udma%u %s(%u) CONTEXT(byte%4lu): %08x\n", + udev->adev_id, name, id, (i + 1) * sizeof(uint32_t), ctx[i]); + + pr_info("**************************************************\n"); +} + +void udma_swap_endian(uint8_t arr[], uint8_t res[], uint32_t res_size) +{ + uint32_t i; + + for (i = 0; i < res_size; i++) + res[i] = arr[res_size - i - 1]; +} + +void udma_init_hugepage(struct udma_dev *dev) +{ + INIT_LIST_HEAD(&dev->hugepage_list); + mutex_init(&dev->hugepage_lock); +} + +void udma_destroy_hugepage(struct udma_dev *dev) +{ + struct udma_hugepage_priv *priv; + + mutex_lock(&dev->hugepage_lock); + list_for_each_entry(priv, &dev->hugepage_list, list) { + dev_info(dev->dev, "unmap_hugepage, 2m_page_num=%u.\n", + priv->va_len >> UDMA_HUGEPAGE_SHIFT); + udma_unpin_k_addr(priv->umem); + vfree(priv->va_base); + kfree(priv); + } + mutex_unlock(&dev->hugepage_lock); + mutex_destroy(&dev->hugepage_lock); +} diff --git a/drivers/ub/urma/hw/udma/udma_common.h b/drivers/ub/urma/hw/udma/udma_common.h new file mode 100644 index 0000000000000000000000000000000000000000..dee92a4186d357a0e19b1d22e2dba697b917dade --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_common.h @@ -0,0 +1,378 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_COMM_H__ +#define __UDMA_COMM_H__ + +#include +#include +#include +#include "udma_ctx.h" +#include "udma_dev.h" + +#define TP_ACK_UDP_SPORT_H_OFFSET 8 +#define UDMA_TPHANDLE_TPID_SHIFT 0xFFFFFF + +struct udma_jetty_grp { + struct ubcore_jetty_group ubcore_jetty_grp; + uint32_t start_jetty_id; + uint32_t next_jetty_id; + uint32_t jetty_grp_id; + uint32_t valid; + struct mutex valid_lock; + refcount_t ae_refcount; + struct completion ae_comp; +}; + +struct udma_jetty_queue { + struct udma_buf buf; + void *kva_curr; + uint32_t id; + void __iomem *db_addr; + void __iomem *dwqe_addr; + uint32_t pi; + uint32_t ci; + uintptr_t *wrid; + spinlock_t lock; + uint32_t max_inline_size; + uint32_t max_sge_num; + uint32_t tid; + bool flush_flag; + uint32_t old_entry_idx; + enum ubcore_transport_mode trans_mode; + struct ubcore_tjetty *rc_tjetty; + bool is_jetty; + uint32_t sqe_bb_cnt; + uint32_t lock_free; /* Support kernel mode lock-free mode */ + uint32_t ta_timeout; /* ms */ + enum ubcore_jetty_state state; + struct udma_context *udma_ctx; + bool non_pin; + struct udma_jetty_grp *jetty_grp; + enum udma_jetty_type jetty_type; +}; + +enum tp_state { + TP_INVALID = 0x0, + TP_VALID = 0x1, + TP_RTS = 0x3, + TP_ERROR = 0x6, +}; + +int pin_queue_addr(struct udma_dev *dev, uint64_t addr, + uint32_t len, struct udma_buf *buf); +void unpin_queue_addr(struct ubcore_umem *umem); + +struct udma_umem_param { + struct ubcore_device *ub_dev; + uint64_t va; + uint64_t len; + union ubcore_umem_flag flag; + bool is_kernel; +}; + +struct udma_ue_index_cmd { + uint16_t ue_idx; + uint8_t rsv[2]; + uint8_t guid[16]; +}; + +struct udma_tp_ctx { + /* Byte4 */ + uint32_t version : 1; + uint32_t tp_mode : 1; + uint32_t trt : 1; + uint32_t wqe_bb_shift : 4; + uint32_t oor_en : 1; + uint32_t tempid : 6; + uint32_t portn : 6; + uint32_t rsvd1 : 12; + /* Byte8 */ + uint32_t wqe_ba_l; + /* Byte12 */ + uint32_t wqe_ba_h : 20; + uint32_t udp_srcport_range : 4; + uint32_t cng_alg_sel : 3; + uint32_t lbi : 1; + uint32_t rsvd4 : 1; + uint32_t vlan_en : 1; + uint32_t mtu : 2; + /* Byte16 */ + uint32_t route_addr_idx : 20; + uint32_t rsvd6 : 12; + /* Byte20 */ + u32 tpn_vtpn : 24; + u32 rsvd7 : 8; + /* Byte24 to Byte28 */ + u32 rsvd8[2]; + /* Byte 32 */ + u32 seid_idx : 16; + u32 sjetty_l : 16; + /* Byte 36 */ + u32 sjetty_h : 4; + u32 tp_wqe_token_id : 20; + u32 tp_wqe_position : 1; + u32 rsv9_l : 7; + /* Byte 40 */ + u32 rsvd9_h : 6; + u32 taack_tpn : 24; + u32 rsvd10 : 2; + /* Byte 44 */ + u32 spray_en : 1; + u32 sr_en : 1; + u32 ack_freq_mode : 1; + u32 route_type : 2; + u32 vl : 4; + u32 dscp : 6; + u32 switch_mp_en : 1; + u32 at_times : 5; + u32 retry_num_init : 3; + u32 at : 5; + u32 rsvd13 : 3; + /* Byte 48 */ + u32 on_flight_size : 16; + u32 hpln : 8; + u32 fl_l : 8; + /* Byte 52 */ + u32 fl_h : 12; + u32 dtpn : 20; + /* Byte 56 */ + u32 rc_tpn : 24; + u32 rc_vl : 4; + u32 tpg_vld : 1; + u32 reorder_cap : 3; + /* Byte 60 */ + u32 reorder_q_shift : 4; + u32 reorder_q_addr_l : 28; + /* Byte 64 */ + u32 reorder_q_addr_h : 24; + u32 tpg_l : 8; + /* Byte 68 */ + u32 tpg_h : 12; + u32 jettyn : 20; + /* Byte 72 */ + u32 dyn_timeout_mode : 1; + u32 base_time : 23; + u32 rsvd15 : 8; + /* Byte 76 */ + u32 tpack_psn : 24; + u32 tpack_rspst : 3; + u32 tpack_rspinfo : 5; + /* Byte 80 */ + u32 tpack_msn : 24; + u32 ack_udp_srcport_l : 8; + /* Byte 84 */ + u32 ack_udp_srcport_h : 8; + u32 max_rcv_psn : 24; + /* Byte 88 */ + u32 scc_token : 19; + u32 poll_db_wait_do : 1; + u32 msg_rty_lp_flg : 1; + u32 retry_cnt : 3; + u32 sq_invld_flg : 1; + u32 wait_ack_timeout : 1; + u32 tx_rtt_caling : 1; + u32 cnp_tx_flag : 1; + u32 sq_db_doing : 1; + u32 tpack_doing : 1; + u32 sack_wait_do : 1; + u32 tpack_wait_do : 1; + /* Byte 92 */ + u16 post_max_idx; + u16 wqe_max_bb_idx; + /* Byte 96 */ + u16 wqe_bb_pi; + u16 wqe_bb_ci; + /* Byte 100 */ + u16 data_udp_srcport; + u16 wqe_msn; + /* Byte 104 */ + u32 cur_req_psn : 24; + u32 tx_ack_psn_err : 1; + u32 poll_db_type : 2; + u32 tx_ack_flg : 1; + u32 tx_sq_err_flg : 1; + u32 scc_retry_type : 2; + u32 flush_cqe_wait_do : 1; + /* Byte 108 */ + u32 wqe_max_psn : 24; + u32 ssc_token_l : 4; + u32 rsvd16 : 4; + /* Byte 112 */ + u32 tx_sq_timer; + /* Byte 116 */ + u32 rtt_timestamp_psn : 24; + u32 rsvd17 : 8; + /* Byte 120 */ + u32 rtt_timestamp : 24; + u32 cnp_timer_l : 8; + /* Byte 124 */ + u32 cnp_timer_h : 16; + u32 max_reorder_id : 16; + /* Byte 128 */ + u16 cur_reorder_id; + u16 wqe_max_msn; + /* Byte 132 */ + u16 post_bb_pi; + u16 post_bb_ci; + /* Byte 136 */ + u32 lr_ae_ind : 1; + u32 rx_cqe_cnt : 16; + u32 reorder_q_si : 13; + u32 rq_err_type_l : 2; + /* Byte 140 */ + u32 rq_err_type_h : 3; + u32 rsvd18 : 2; + u32 rsvd19 : 27; + /* Byte 144 */ + u32 req_seq; + /* Byte 148 */ + uint32_t req_ce_seq; + /* Byte 152 */ + u32 req_cmp_lrb_indx : 12; + u32 req_lrb_indx : 12; + u32 req_lrb_indx_vld : 1; + u32 rx_req_psn_err : 1; + u32 rx_req_last_optype : 3; + u32 rx_req_fake_flg : 1; + u32 rsvd20 : 2; + /* Byte 156 */ + uint16_t jfr_wqe_idx; + uint16_t rx_req_epsn_l; + /* Byte 160 */ + uint32_t rx_req_epsn_h : 8; + uint32_t rx_req_reduce_code : 8; + uint32_t rx_req_msn_l : 16; + /* Byte 164 */ + uint32_t rx_req_msn_h : 8; + uint32_t jfr_wqe_rnr : 1; + uint32_t jfr_wqe_rnr_timer : 5; + uint32_t rsvd21 : 2; + uint32_t jfr_wqe_cnt : 16; + /* Byte 168 */ + uint32_t max_reorder_q_idx : 13; + uint32_t rsvd22 : 3; + uint32_t reorder_q_ei : 13; + uint32_t rx_req_last_elr_flg : 1; + uint32_t rx_req_last_elr_err_type_l : 2; + /* Byte172 */ + uint32_t rx_req_last_elr_err_type_h : 3; + uint32_t rx_req_last_op : 1; + uint32_t jfrx_jetty : 1; + uint32_t jfrx_jfcn_l : 16; + uint32_t jfrx_jfcn_h : 4; + uint32_t jfrx_jfrn_l : 7; + /* Byte176 */ + u32 jfrx_jfrn_h1 : 9; + u32 jfrx_jfrn_h2 : 4; + u32 rq_timer_l : 19; + /* Byte180 */ + u32 rq_timer_h : 13; + u32 rq_at : 5; + u32 wait_cqe_timeout : 1; + u32 rsvd23 : 13; + /* Byte184 */ + u32 rx_sq_timer; + /* Byte188 */ + u32 tp_st : 3; + u32 rsvd24 : 4; + u32 ls_ae_ind : 1; + u32 retry_msg_psn : 24; + /* Byte192 */ + u32 retry_msg_fpsn : 24; + u32 rsvd25 : 8; + /* Byte196 */ + u16 retry_wqebb_idx; + u16 retry_msg_msn; + /* Byte200 */ + u32 ack_rcv_seq; + /* Byte204 */ + u32 rtt : 24; + u32 dup_sack_cnt : 8; + /* Byte208 */ + u32 sack_max_rcv_psn : 24; + u32 rsvd26 : 7; + u32 rx_ack_flg : 1; + /* Byte212 */ + u32 rx_ack_msn : 16; + u32 sack_lrb_indx : 12; + u32 rx_fake_flg : 1; + u32 rx_rtt_caling : 1; + u32 rx_ack_psn_err : 1; + u32 sack_lrb_indx_vld : 1; + /* Byte216 */ + u32 rx_ack_epsn : 24; + u32 rsvd27 : 8; + /* Byte220 */ + u32 max_retry_psn : 24; + u32 retry_reorder_id_l : 8; + /* Byte224 */ + u32 retry_reorder_id_h : 8; + u32 rsvd28 : 8; + u32 rsvd29 : 16; + /* Byte228 to Byte256 */ + u32 scc_data[8]; +}; + +struct ubcore_umem *udma_umem_get(struct udma_umem_param *param); +void udma_umem_release(struct ubcore_umem *umem, bool is_kernel); +void udma_init_udma_table(struct udma_table *table, uint32_t max, uint32_t min); +void udma_init_udma_table_mutex(struct xarray *table, struct mutex *udma_mutex); +void udma_destroy_npu_cb_table(struct udma_dev *dev); +void udma_destroy_udma_table(struct udma_dev *dev, struct udma_table *table, + const char *table_name); +void udma_destroy_eid_table(struct udma_dev *udma_dev); +void udma_dfx_store_id(struct udma_dev *udma_dev, struct udma_dfx_entity *entity, + uint32_t id, const char *name); +void udma_dfx_delete_id(struct udma_dev *udma_dev, struct udma_dfx_entity *entity, + uint32_t id); +int udma_alloc_normal_buf(struct udma_dev *udma_dev, size_t memory_size, struct udma_buf *buf); +void udma_free_normal_buf(struct udma_dev *udma_dev, size_t memory_size, struct udma_buf *buf); +int udma_k_alloc_buf(struct udma_dev *dev, struct udma_buf *buf); +void udma_k_free_buf(struct udma_dev *dev, struct udma_buf *buf); +void *udma_alloc_iova(struct udma_dev *udma_dev, size_t memory_size, dma_addr_t *addr); +void udma_free_iova(struct udma_dev *udma_dev, size_t memory_size, void *kva_or_slot, + dma_addr_t addr); + +static inline void udma_write64(struct udma_dev *udma_dev, + uint64_t *val, void __iomem *dest) +{ + writeq(*val, dest); +} + +static inline void udma_alloc_kernel_db(struct udma_dev *dev, + struct udma_jetty_queue *queue) +{ + queue->dwqe_addr = dev->k_db_base + JETTY_DSQE_OFFSET + + UDMA_HW_PAGE_SIZE * queue->id; + queue->db_addr = queue->dwqe_addr + UDMA_DOORBELL_OFFSET; +} + +static inline void *get_buf_entry(struct udma_buf *buf, uint32_t n) +{ + uint32_t entry_index = n & (buf->entry_cnt - 1); + + return (char *)buf->kva + (entry_index * buf->entry_size); +} + +static inline uint8_t to_ta_timeout(uint32_t err_timeout) +{ +#define TA_TIMEOUT_DIVISOR 8 + return err_timeout / TA_TIMEOUT_DIVISOR; +} + +static inline uint64_t udma_cal_npages(uint64_t va, uint64_t len) +{ + return (ALIGN(va + len, PAGE_SIZE) - ALIGN_DOWN(va, PAGE_SIZE)) / PAGE_SIZE; +} + +int udma_query_ue_idx(struct ubcore_device *ub_dev, struct ubcore_devid *devid, + uint16_t *ue_idx); +void udma_dfx_ctx_print(struct udma_dev *udev, const char *name, uint32_t id, uint32_t len, + uint32_t *ctx); +void udma_swap_endian(uint8_t arr[], uint8_t res[], uint32_t res_size); + +void udma_init_hugepage(struct udma_dev *dev); +void udma_destroy_hugepage(struct udma_dev *dev); + +#endif /* __UDMA_COMM_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_ctl.c b/drivers/ub/urma/hw/udma/udma_ctl.c new file mode 100644 index 0000000000000000000000000000000000000000..af0568f3ce746139716e14f0c9a20e7398c60e66 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_ctl.c @@ -0,0 +1,1449 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt + +#include +#include +#include +#include "udma_common.h" +#include "udma_dev.h" +#include +#include "udma_cmd.h" +#include "udma_jetty.h" +#include "udma_segment.h" +#include "udma_jfs.h" +#include "udma_jfc.h" +#include "udma_db.h" +#include "udma_ctrlq_tp.h" +#include +#include "udma_def.h" + +const char *udma_cqe_aux_info_type_str[] = { + "TPP2TQEM_WR_CNT", + "DEVICE_RAS_STATUS_2", + "RXDMA_WR_PAYL_AXI_ERR", + "RXDMA_HEAD_SPLIT_ERR_FLAG0", + "RXDMA_HEAD_SPLIT_ERR_FLAG1", + "RXDMA_HEAD_SPLIT_ERR_FLAG2", + "RXDMA_HEAD_SPLIT_ERR_FLAG3", + "TP_RCP_INNER_ALM", + "TWP_AE_DFX", + "PA_OUT_PKT_ERR_CNT", + "TP_DAM_AXI_ALARM", + "TP_DAM_VFT_BT_ALARM", + "TP_EUM_AXI_ALARM", + "TP_EUM_VFT_BT_ALARM", + "TP_TPMM_AXI_ALARM", + "TP_TPMM_VFT_BT_ALARM", + "TP_TPGCM_AXI_ALARM", + "TP_TPGCM_VFT_BT_ALARM", + "TWP_ALM", + "TP_RWP_INNER_ALM", + "TWP_DFX21", + "LQC_TA_RNR_TANACK_CNT", + "FVT", + "RQMT0", + "RQMT1", + "RQMT2", + "RQMT3", + "RQMT4", + "RQMT5", + "RQMT6", + "RQMT7", + "RQMT8", + "RQMT9", + "RQMT10", + "RQMT11", + "RQMT12", + "RQMT13", + "RQMT14", + "RQMT15", + "PROC_ERROR_ALM", + "LQC_TA_TIMEOUT_TAACK_CNT", + "TP_RRP_ERR_FLG_0", +}; + +const char *udma_ae_aux_info_type_str[] = { + "TP_RRP_FLUSH_TIMER_PKT_CNT", + "TPP_DFX5", + "TWP_AE_DFX", + "TP_RRP_ERR_FLG_0", + "TP_RRP_ERR_FLG_1", + "TP_RWP_INNER_ALM", + "TP_RCP_INNER_ALM", + "LQC_TA_TQEP_WQE_ERR", + "LQC_TA_CQM_CQE_INNER_ALARM", +}; + +static int udma_get_sq_buf_ex(struct udma_dev *dev, struct udma_jetty_queue *sq, + struct udma_jfs_cfg_ex *cfg_ex) +{ + struct ubcore_jfs_cfg *jfs_cfg; + uint32_t wqe_bb_depth; + uint32_t sqe_bb_cnt; + uint32_t size; + + jfs_cfg = &cfg_ex->base_cfg; + + if (!jfs_cfg->flag.bs.lock_free) + spin_lock_init(&sq->lock); + sq->max_inline_size = jfs_cfg->max_inline_data; + sq->max_sge_num = jfs_cfg->max_sge; + sq->tid = dev->tid; + sq->lock_free = jfs_cfg->flag.bs.lock_free; + + sqe_bb_cnt = sq_cal_wqebb_num(SQE_WRITE_NOTIFY_CTL_LEN, jfs_cfg->max_sge); + if (sqe_bb_cnt > MAX_WQEBB_NUM) + sqe_bb_cnt = MAX_WQEBB_NUM; + sq->sqe_bb_cnt = sqe_bb_cnt; + + wqe_bb_depth = roundup_pow_of_two(sqe_bb_cnt * jfs_cfg->depth); + sq->buf.entry_size = UDMA_JFS_WQEBB_SIZE; + size = ALIGN(wqe_bb_depth * sq->buf.entry_size, UDMA_HW_PAGE_SIZE); + sq->buf.entry_cnt = size >> WQE_BB_SIZE_SHIFT; + + if (size != cfg_ex->cstm_cfg.sq.buff_size) { + dev_err(dev->dev, "buff size is wrong, buf size = %u.\n", size); + return -EINVAL; + } + + if (cfg_ex->cstm_cfg.sq.buff == 0) { + dev_err(dev->dev, "cstm_cfg sq buff is wrong.\n"); + return -EINVAL; + } + + sq->buf.addr = (dma_addr_t)(uintptr_t)phys_to_virt((uint64_t) + (uintptr_t)cfg_ex->cstm_cfg.sq.buff); + if (sq->buf.addr == 0) { + dev_err(dev->dev, "sq buff addr is wrong.\n"); + return -EINVAL; + } + + sq->buf.kva = (void *)(uintptr_t)sq->buf.addr; + + sq->wrid = kcalloc(1, sq->buf.entry_cnt * sizeof(uint64_t), GFP_KERNEL); + if (!sq->wrid) { + sq->buf.kva = NULL; + sq->buf.addr = 0; + dev_err(dev->dev, + "failed to alloc wrid for jfs id = %u when entry cnt = %u.\n", + sq->id, sq->buf.entry_cnt); + return -ENOMEM; + } + + udma_alloc_kernel_db(dev, sq); + sq->kva_curr = sq->buf.kva; + + sq->trans_mode = jfs_cfg->trans_mode; + + return 0; +} + +static int udma_get_jfs_buf_ex(struct udma_dev *dev, struct udma_jfs *jfs, + struct udma_jfs_cfg_ex *cfg_ex) +{ + int ret; + + jfs->jfs_addr = (uintptr_t)&jfs->sq; + + ret = udma_get_sq_buf_ex(dev, &jfs->sq, cfg_ex); + if (ret) + dev_err(dev->dev, + "failed to get sq buf in jfs process, ret = %d.\n", ret); + + return ret; +} + +static struct ubcore_jfs *udma_create_jfs_ex(struct ubcore_device *ub_dev, + struct udma_jfs_cfg_ex *cfg_ex) +{ + struct ubcore_jfs_cfg *cfg = &cfg_ex->base_cfg; + struct udma_dev *dev = to_udma_dev(ub_dev); + struct ubase_mbx_attr attr = {}; + struct udma_jetty_ctx ctx = {}; + struct udma_jfs *jfs; + int ret; + + ret = udma_verify_jfs_param(dev, cfg, true); + if (ret) + return NULL; + + jfs = kcalloc(1, sizeof(*jfs), GFP_KERNEL); + if (!jfs) + return NULL; + + dev_info(dev->dev, "start alloc id!\n"); + ret = udma_alloc_jetty_id(dev, &jfs->sq.id, &dev->caps.jetty); + if (ret) { + dev_err(dev->dev, "alloc JFS id failed, ret = %d.\n", ret); + goto err_alloc_jfsn; + } + jfs->ubcore_jfs.jfs_id.id = jfs->sq.id; + jfs->ubcore_jfs.jfs_cfg = *cfg; + jfs->ubcore_jfs.ub_dev = ub_dev; + jfs->ubcore_jfs.uctx = NULL; + jfs->ubcore_jfs.jfae_handler = cfg_ex->jfae_handler; + jfs->mode = UDMA_KERNEL_STARS_JFS_TYPE; + + ret = xa_err(xa_store(&dev->jetty_table.xa, jfs->sq.id, &jfs->sq, GFP_KERNEL)); + if (ret) { + dev_err(dev->dev, "store jfs sq(%u) failed, ret = %d.\n", + jfs->sq.id, ret); + goto err_store_jfs_sq; + } + + dev_info(dev->dev, "start get stars jfs buf!\n"); + ret = udma_get_jfs_buf_ex(dev, jfs, cfg_ex); + if (ret) + goto err_alloc_jfs_id; + + udma_set_query_flush_time(&jfs->sq, cfg->err_timeout); + jfs->sq.state = UBCORE_JETTY_STATE_READY; + udma_init_jfsc(dev, cfg, jfs, &ctx); + attr.tag = jfs->sq.id; + attr.op = UDMA_CMD_CREATE_JFS_CONTEXT; + ret = post_mailbox_update_ctx(dev, &ctx, sizeof(ctx), &attr); + if (ret) { + dev_err(dev->dev, "failed to upgrade JFSC, ret = %d.\n", ret); + goto err_update_ctx; + } + + refcount_set(&jfs->ae_refcount, 1); + init_completion(&jfs->ae_comp); + + if (dfx_switch) + udma_dfx_store_jfs_id(dev, jfs); + + dev_info(dev->dev, "create stars jfs success!\n"); + + return &jfs->ubcore_jfs; + +err_update_ctx: + kfree(jfs->sq.wrid); +err_alloc_jfs_id: + xa_erase(&dev->jetty_table.xa, jfs->sq.id); +err_store_jfs_sq: + udma_adv_id_free(&dev->jetty_table.bitmap_table, jfs->sq.id, false); +err_alloc_jfsn: + kfree(jfs); + return NULL; +} + +static int udma_create_jfs_ops_ex(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out) +{ + struct udma_dev *udev = to_udma_dev(dev); + struct udma_jfs_cfg_ex cfg_ex; + struct ubcore_jfs *jfs; + + if (udma_check_base_param(in->addr, in->len, sizeof(struct udma_jfs_cfg_ex)) || + udma_check_base_param(out->addr, out->len, sizeof(struct ubcore_jfs *))) { + dev_err(udev->dev, "param invalid in create jfs, in_len = %u, out_len = %u.\n", + in->len, out->len); + return -EINVAL; + } + + memcpy(&cfg_ex, (void *)(uintptr_t)in->addr, sizeof(struct udma_jfs_cfg_ex)); + + jfs = udma_create_jfs_ex(dev, &cfg_ex); + if (jfs == NULL) + return -EFAULT; + + memcpy((void *)(uintptr_t)out->addr, &jfs, sizeof(struct ubcore_jfs *)); + + return 0; +} + +static int udma_delete_jfs_ops_ex(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out) +{ + struct udma_dev *udev = to_udma_dev(dev); + struct ubcore_jfs *jfs; + + if (udma_check_base_param(in->addr, in->len, sizeof(struct ubcore_jfs *))) { + dev_err(udev->dev, "parameter invalid in delete jfs, len = %u.\n", + in->len); + return -EFAULT; + } + memcpy(&jfs, (void *)(uintptr_t)in->addr, sizeof(struct ubcore_jfs *)); + if (jfs == NULL) + return -EINVAL; + + if (udma_destroy_jfs(jfs)) + return -EFAULT; + + return 0; +} + +static int udma_get_jfc_buf_ex(struct udma_dev *dev, + struct udma_jfc *jfc, + struct udma_jfc_cfg_ex *cfg_ex) +{ + uint32_t size; + int ret = 0; + + if (!jfc->lock_free) + spin_lock_init(&jfc->lock); + jfc->buf.entry_size = dev->caps.cqe_size; + jfc->tid = dev->tid; + size = jfc->buf.entry_size * jfc->buf.entry_cnt; + + if (size != cfg_ex->cstm_cfg.cq.buff_size) { + dev_err(dev->dev, "cqe buff size is wrong, buf size = %u.\n", size); + return -EINVAL; + } + + jfc->buf.addr = (dma_addr_t)(uintptr_t)cfg_ex->cstm_cfg.cq.buff; + + if (jfc->buf.addr == 0) { + dev_err(dev->dev, "cq buff addr is wrong.\n"); + return -EINVAL; + } + + jfc->buf.kva = (void *)(uintptr_t)jfc->buf.addr; + + ret = udma_alloc_sw_db(dev, &jfc->db, UDMA_JFC_TYPE_DB); + if (ret) { + dev_err(dev->dev, "failed to alloc sw db for jfc(%u).\n", jfc->jfcn); + return -ENOMEM; + } + + return ret; +} + +static struct ubcore_jfc *udma_create_jfc_ex(struct ubcore_device *ubcore_dev, + struct udma_jfc_cfg_ex *cfg_ex) +{ + struct udma_dev *dev = to_udma_dev(ubcore_dev); + struct ubcore_jfc_cfg *cfg = &cfg_ex->base_cfg; + unsigned long flags_store; + unsigned long flags_erase; + struct udma_jfc *jfc; + int ret; + + jfc = kzalloc(sizeof(struct udma_jfc), GFP_KERNEL); + if (!jfc) + return NULL; + + jfc->arm_sn = 1; + jfc->buf.entry_cnt = cfg->depth ? roundup_pow_of_two(cfg->depth) : cfg->depth; + + ret = udma_check_jfc_cfg(dev, jfc, &cfg_ex->base_cfg); + if (ret) + goto err_check_cfg; + + ret = udma_id_alloc_auto_grow(dev, &dev->jfc_table.ida_table, &jfc->jfcn); + if (ret) + goto err_alloc_jfc_id; + + udma_init_jfc_param(cfg, jfc); + jfc->base.ub_dev = ubcore_dev; + jfc->base.uctx = NULL; + jfc->base.jfae_handler = cfg_ex->jfae_handler; + jfc->base.jfce_handler = cfg_ex->jfce_handler; + jfc->mode = UDMA_KERNEL_STARS_JFC_TYPE; + + xa_lock_irqsave(&dev->jfc_table.xa, flags_store); + ret = xa_err(__xa_store(&dev->jfc_table.xa, jfc->jfcn, jfc, GFP_ATOMIC)); + xa_unlock_irqrestore(&dev->jfc_table.xa, flags_store); + if (ret) { + dev_err(dev->dev, + "failed to stored jfc id to jfc_table, jfcn: %u.\n", + jfc->jfcn); + goto err_store_jfcn; + } + + ret = udma_get_jfc_buf_ex(dev, jfc, cfg_ex); + if (ret) + goto err_get_jfc_buf; + + ret = udma_post_create_jfc_mbox(dev, jfc); + if (ret) + goto err_alloc_cqc; + + refcount_set(&jfc->event_refcount, 1); + + init_completion(&jfc->event_comp); + + if (dfx_switch) + udma_dfx_store_id(dev, &dev->dfx_info->jfc, jfc->jfcn, "jfc"); + + return &jfc->base; + +err_alloc_cqc: + udma_free_sw_db(dev, &jfc->db); +err_get_jfc_buf: + xa_lock_irqsave(&dev->jfc_table.xa, flags_erase); + __xa_erase(&dev->jfc_table.xa, jfc->jfcn); + xa_unlock_irqrestore(&dev->jfc_table.xa, flags_erase); +err_store_jfcn: + udma_id_free(&dev->jfc_table.ida_table, jfc->jfcn); +err_alloc_jfc_id: +err_check_cfg: + kfree(jfc); + return NULL; +} + +static int udma_create_jfc_ops_ex(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out) +{ + struct udma_dev *udev = to_udma_dev(dev); + struct udma_jfc_cfg_ex cfg_ex; + struct ubcore_jfc *jfc; + + if (udma_check_base_param(in->addr, in->len, sizeof(struct udma_jfc_cfg_ex)) || + udma_check_base_param(out->addr, out->len, sizeof(struct ubcore_jfc *))) { + dev_err(udev->dev, "input parameter invalid in create jfc, in_len = %u, out_len = %u.\n", + in->len, out->len); + return -EINVAL; + } + + memcpy(&cfg_ex, (void *)(uintptr_t)in->addr, + min(in->len, sizeof(struct udma_jfc_cfg_ex))); + + jfc = udma_create_jfc_ex(dev, &cfg_ex); + if (jfc == NULL) + return -EFAULT; + + memcpy((void *)(uintptr_t)out->addr, &jfc, sizeof(struct ubcore_jfc *)); + + return 0; +} + +static int udma_delete_jfc_ops_ex(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out) +{ + struct udma_dev *udev = to_udma_dev(dev); + struct ubcore_jfc *jfc; + + if (udma_check_base_param(in->addr, in->len, sizeof(struct ubcore_jfc *))) { + dev_err(udev->dev, "parameter invalid in delete jfc, len = %u.\n", + in->len); + return -EINVAL; + } + + memcpy(&jfc, (void *)(uintptr_t)in->addr, + min(in->len, sizeof(struct ubcore_jfc *))); + if (jfc == NULL) + return -EINVAL; + + if (udma_destroy_jfc(jfc)) + return -EFAULT; + + return 0; +} + +static int udma_set_cqe_ex(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out) +{ + struct udma_dev *udev = to_udma_dev(dev); + struct udma_ex_jfc_addr *jfc_addr; + struct udma_set_cqe_ex cqe_ex; + uint32_t cq_depth; + + if (udma_check_base_param(in->addr, in->len, sizeof(struct udma_set_cqe_ex))) { + dev_err(udev->dev, "parameter invalid in set cqe, len = %u.\n", + in->len); + return -EINVAL; + } + + memcpy(&cqe_ex, (void *)(uintptr_t)in->addr, + min(in->len, sizeof(struct udma_set_cqe_ex))); + + if (cqe_ex.jfc_type != UDMA_STARS_JFC_TYPE && + cqe_ex.jfc_type != UDMA_CCU_JFC_TYPE) { + dev_err(udev->dev, "invalid jfc type, mode = %u.\n", cqe_ex.jfc_type); + return -EINVAL; + } + + if (cqe_ex.addr == 0) { + dev_err(udev->dev, "cq addr is wrong in set cqe.\n"); + return -EINVAL; + } + + cq_depth = cqe_ex.len / udev->caps.cqe_size; + if (cq_depth < UDMA_JFC_DEPTH_MIN || cq_depth > udev->caps.jfc.depth || + (cqe_ex.len % udev->caps.cqe_size) != 0 || + cq_depth != roundup_pow_of_two(cq_depth)) { + dev_err(udev->dev, "cq buff size is wrong in set cqe, size = %u.\n", + cqe_ex.len); + return -EINVAL; + } + + jfc_addr = &udev->cq_addr_array[cqe_ex.jfc_type]; + jfc_addr->cq_addr = cqe_ex.addr; + jfc_addr->cq_len = cqe_ex.len; + + return 0; +} + +static int udma_query_ue_info_ex(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out) +{ + struct udma_dev *udev = to_udma_dev(dev); + struct udma_ue_info_ex info = {}; + + if (udma_check_base_param(out->addr, out->len, sizeof(struct udma_ue_info_ex))) { + dev_err(udev->dev, "parameter invalid in query ue, len = %u.\n", + out->len); + return -EINVAL; + } + + info.chip_id = udev->chip_id; + info.die_id = udev->die_id; + info.dwqe_addr = udev->db_base + JETTY_DSQE_OFFSET; + info.db_base_addr = info.dwqe_addr + UDMA_DOORBELL_OFFSET; + info.ue_id = udev->ue_id; + info.register_base_addr = udev->db_base; + info.offset_len = PAGE_SIZE; + + memcpy((void *)(uintptr_t)out->addr, &info, sizeof(struct udma_ue_info_ex)); + + return 0; +} + +static int udma_ctrlq_query_tp_sport(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out) +{ + struct udma_tp_sport_out tp_sport_out = {}; + struct udma_tp_sport_in tp_sport_in = {}; + struct udma_dev *udev = to_udma_dev(dev); + struct ubase_cmd_mailbox *mailbox = NULL; + struct ubase_mbx_attr mbox_attr = {}; + struct udma_tp_ctx *tpc; + + if (udma_check_base_param(out->addr, out->len, sizeof(struct udma_tp_sport_out)) || + udma_check_base_param(in->addr, in->len, sizeof(struct udma_tp_sport_in))) { + dev_err(udev->dev, "parameter invalid in query tp sport, in_len = %u, out_len = %u.\n", + in->len, out->len); + return -EINVAL; + } + + if (udev->is_ue) { + dev_err(udev->dev, "ue is not supported.\n"); + return -EINVAL; + } + + memcpy(&tp_sport_in, (void *)(uintptr_t)in->addr, sizeof(struct udma_tp_sport_in)); + + mbox_attr.tag = tp_sport_in.tpn; + mbox_attr.op = UDMA_CMD_QUERY_TP_CONTEXT; + mailbox = udma_mailbox_query_ctx(udev, &mbox_attr); + if (!mailbox) + return -ENOMEM; + + tpc = (struct udma_tp_ctx *)mailbox->buf; + + tp_sport_out.ack_udp_srcport = tpc->ack_udp_srcport_h << TP_ACK_UDP_SPORT_H_OFFSET | + tpc->ack_udp_srcport_l; + tp_sport_out.data_udp_srcport = tpc->data_udp_srcport; + + memcpy((void *)(uintptr_t)out->addr, &tp_sport_out, out->len); + + udma_free_cmd_mailbox(udev, mailbox); + + return 0; +} + +static void dump_cqe_client_loc_len_err_aux_info(struct udma_dev *dev, + struct udma_cqe_aux_info_out *aux_info_out, + struct udma_cmd_query_cqe_aux_info *info) +{ + enum udma_cqe_aux_info_type type[] = { + TPP2TQEM_WR_CNT, + DEVICE_RAS_STATUS_2, + }; + uint32_t aux_info_num = ARRAY_SIZE(type); + int i; + + if (aux_info_out->aux_info_type != NULL && + aux_info_out->aux_info_value != NULL && + aux_info_out->aux_info_num >= aux_info_num) { + for (i = 0; i < aux_info_num; i++) { + aux_info_out->aux_info_type[i] = type[i]; + aux_info_out->aux_info_value[i] = info->cqe_aux_info[type[i]]; + } + aux_info_out->aux_info_num = aux_info_num; + } + + for (i = 0; i < aux_info_num; i++) + dev_info(dev->dev, "%s\t0x%08x\n", + udma_cqe_aux_info_type_str[type[i]], info->cqe_aux_info[type[i]]); +} + +static void dump_cqe_client_loc_access_err_aux_info(struct udma_dev *dev, + struct udma_cqe_aux_info_out *aux_info_out, + struct udma_cmd_query_cqe_aux_info *info) +{ + enum udma_cqe_aux_info_type type[] = { + RXDMA_WR_PAYL_AXI_ERR, + RXDMA_HEAD_SPLIT_ERR_FLAG0, + RXDMA_HEAD_SPLIT_ERR_FLAG1, + RXDMA_HEAD_SPLIT_ERR_FLAG2, + RXDMA_HEAD_SPLIT_ERR_FLAG3, + TP_RCP_INNER_ALM_FOR_CQE, + TWP_AE_DFX_FOR_CQE, + PA_OUT_PKT_ERR_CNT, + TP_DAM_AXI_ALARM, + TP_DAM_VFT_BT_ALARM, + TP_EUM_AXI_ALARM, + TP_EUM_VFT_BT_ALARM, + TP_TPMM_AXI_ALARM, + TP_TPMM_VFT_BT_ALARM, + TP_TPGCM_AXI_ALARM, + TP_TPGCM_VFT_BT_ALARM, + DEVICE_RAS_STATUS_2, + TWP_ALM, + }; + uint32_t aux_info_num = ARRAY_SIZE(type); + int i; + + if (aux_info_out->aux_info_type != NULL && + aux_info_out->aux_info_value != NULL && + aux_info_out->aux_info_num >= aux_info_num) { + for (i = 0; i < aux_info_num; i++) { + aux_info_out->aux_info_type[i] = type[i]; + aux_info_out->aux_info_value[i] = info->cqe_aux_info[type[i]]; + } + aux_info_out->aux_info_num = aux_info_num; + } + + for (i = 0; i < aux_info_num; i++) + dev_info(dev->dev, "%s\t0x%08x\n", + udma_cqe_aux_info_type_str[type[i]], info->cqe_aux_info[type[i]]); +} + +static void dump_cqe_client_rem_resp_len_err_aux_info(struct udma_dev *dev, + struct udma_cqe_aux_info_out *aux_info_out, + struct udma_cmd_query_cqe_aux_info *info) +{ + enum udma_cqe_aux_info_type type[] = { + TP_RWP_INNER_ALM_FOR_CQE, + }; + uint32_t aux_info_num = ARRAY_SIZE(type); + int i; + + if (aux_info_out->aux_info_type != NULL && + aux_info_out->aux_info_value != NULL && + aux_info_out->aux_info_num >= aux_info_num) { + for (i = 0; i < aux_info_num; i++) { + aux_info_out->aux_info_type[i] = type[i]; + aux_info_out->aux_info_value[i] = info->cqe_aux_info[type[i]]; + } + aux_info_out->aux_info_num = aux_info_num; + } + + for (i = 0; i < aux_info_num; i++) + dev_info(dev->dev, "%s\t0x%08x\n", + udma_cqe_aux_info_type_str[type[i]], info->cqe_aux_info[type[i]]); +} + +static void +dump_cqe_client_rem_access_abort_err_aux_info(struct udma_dev *dev, + struct udma_cqe_aux_info_out *aux_info_out, + struct udma_cmd_query_cqe_aux_info *info) +{ + enum udma_cqe_aux_info_type type[] = { + RXDMA_WR_PAYL_AXI_ERR, + RXDMA_HEAD_SPLIT_ERR_FLAG0, + RXDMA_HEAD_SPLIT_ERR_FLAG1, + RXDMA_HEAD_SPLIT_ERR_FLAG2, + RXDMA_HEAD_SPLIT_ERR_FLAG3, + TP_RCP_INNER_ALM_FOR_CQE, + TP_RRP_ERR_FLG_0_FOR_CQE, + TPP2TQEM_WR_CNT, + TWP_DFX21 + }; + uint32_t aux_info_num = ARRAY_SIZE(type); + int i; + + if (aux_info_out->aux_info_type != NULL && + aux_info_out->aux_info_value != NULL && + aux_info_out->aux_info_num >= aux_info_num) { + for (i = 0; i < aux_info_num; i++) { + aux_info_out->aux_info_type[i] = type[i]; + aux_info_out->aux_info_value[i] = info->cqe_aux_info[type[i]]; + } + aux_info_out->aux_info_num = aux_info_num; + } + + for (i = 0; i < aux_info_num; i++) + dev_info(dev->dev, "%s\t0x%08x\n", + udma_cqe_aux_info_type_str[type[i]], info->cqe_aux_info[type[i]]); +} + +static void dump_cqe_client_ack_timeout_err_aux_info(struct udma_dev *dev, + struct udma_cqe_aux_info_out *aux_info_out, + struct udma_cmd_query_cqe_aux_info *info) +{ + enum udma_cqe_aux_info_type type[] = { + LQC_TA_TIMEOUT_TAACK_CNT, + }; + uint32_t aux_info_num = ARRAY_SIZE(type); + int i; + + if (aux_info_out->aux_info_type != NULL && + aux_info_out->aux_info_value != NULL && + aux_info_out->aux_info_num >= aux_info_num) { + for (i = 0; i < aux_info_num; i++) { + aux_info_out->aux_info_type[i] = type[i]; + aux_info_out->aux_info_value[i] = info->cqe_aux_info[type[i]]; + } + aux_info_out->aux_info_num = aux_info_num; + } + + for (i = 0; i < aux_info_num; i++) + dev_info(dev->dev, "%s\t0x%08x\n", + udma_cqe_aux_info_type_str[type[i]], info->cqe_aux_info[type[i]]); +} + +static void +dump_cqe_client_rnr_retry_cnt_exc_err_aux_info(struct udma_dev *dev, + struct udma_cqe_aux_info_out *aux_info_out, + struct udma_cmd_query_cqe_aux_info *info) +{ + enum udma_cqe_aux_info_type type[] = { + LQC_TA_RNR_TANACK_CNT, + FVT, + RQMT0, + RQMT1, + RQMT2, + RQMT3, + RQMT4, + RQMT5, + RQMT6, + RQMT7, + RQMT8, + RQMT9, + RQMT10, + RQMT11, + RQMT12, + RQMT13, + RQMT14, + RQMT15, + PROC_ERROR_ALM, + }; + uint32_t aux_info_num = ARRAY_SIZE(type); + int i; + + if (aux_info_out->aux_info_type != NULL && + aux_info_out->aux_info_value != NULL && + aux_info_out->aux_info_num >= aux_info_num) { + for (i = 0; i < aux_info_num; i++) { + aux_info_out->aux_info_type[i] = type[i]; + aux_info_out->aux_info_value[i] = info->cqe_aux_info[type[i]]; + } + aux_info_out->aux_info_num = aux_info_num; + } + + for (i = 0; i < aux_info_num; i++) + dev_info(dev->dev, "%s\t0x%08x\n", + udma_cqe_aux_info_type_str[type[i]], info->cqe_aux_info[type[i]]); +} + +static void dump_cqe_server_loc_access_err_aux_info(struct udma_dev *dev, + struct udma_cqe_aux_info_out *aux_info_out, + struct udma_cmd_query_cqe_aux_info *info) +{ + enum udma_cqe_aux_info_type type[] = { + TP_RWP_INNER_ALM_FOR_CQE, + RXDMA_WR_PAYL_AXI_ERR, + RXDMA_HEAD_SPLIT_ERR_FLAG0, + RXDMA_HEAD_SPLIT_ERR_FLAG1, + RXDMA_HEAD_SPLIT_ERR_FLAG2, + RXDMA_HEAD_SPLIT_ERR_FLAG3, + TP_RCP_INNER_ALM_FOR_CQE, + TP_RRP_ERR_FLG_0_FOR_CQE, + }; + uint32_t aux_info_num = ARRAY_SIZE(type); + int i; + + if (aux_info_out->aux_info_type != NULL && + aux_info_out->aux_info_value != NULL && + aux_info_out->aux_info_num >= aux_info_num) { + for (i = 0; i < aux_info_num; i++) { + aux_info_out->aux_info_type[i] = type[i]; + aux_info_out->aux_info_value[i] = info->cqe_aux_info[type[i]]; + } + aux_info_out->aux_info_num = aux_info_num; + } + + for (i = 0; i < aux_info_num; i++) + dev_info(dev->dev, "%s\t0x%08x\n", + udma_cqe_aux_info_type_str[type[i]], info->cqe_aux_info[type[i]]); +} + +static void dump_cqe_server_loc_len_err_aux_info(struct udma_dev *dev, + struct udma_cqe_aux_info_out *aux_info_out, + struct udma_cmd_query_cqe_aux_info *info) +{ + enum udma_cqe_aux_info_type type[] = { + TP_RWP_INNER_ALM_FOR_CQE, + }; + uint32_t aux_info_num = ARRAY_SIZE(type); + int i; + + if (aux_info_out->aux_info_type != NULL && + aux_info_out->aux_info_value != NULL && + aux_info_out->aux_info_num >= aux_info_num) { + for (i = 0; i < aux_info_num; i++) { + aux_info_out->aux_info_type[i] = type[i]; + aux_info_out->aux_info_value[i] = info->cqe_aux_info[type[i]]; + } + aux_info_out->aux_info_num = aux_info_num; + } + + for (i = 0; i < aux_info_num; i++) + dev_info(dev->dev, "%s\t0x%08x\n", + udma_cqe_aux_info_type_str[type[i]], info->cqe_aux_info[type[i]]); +} + +static void dump_cqe_all_aux_info(struct udma_dev *dev, + struct udma_cqe_aux_info_out *aux_info_out, + struct udma_cmd_query_cqe_aux_info *info) +{ + int i; + + if (aux_info_out->aux_info_type != NULL && + aux_info_out->aux_info_value != NULL && + aux_info_out->aux_info_num >= MAX_CQE_AUX_INFO_TYPE_NUM) { + for (i = 0; i < MAX_CQE_AUX_INFO_TYPE_NUM; i++) { + aux_info_out->aux_info_type[i] = i; + aux_info_out->aux_info_value[i] = info->cqe_aux_info[i]; + } + aux_info_out->aux_info_num = MAX_CQE_AUX_INFO_TYPE_NUM; + } + + for (i = 0; i < MAX_CQE_AUX_INFO_TYPE_NUM; i++) + dev_info(dev->dev, "%s\t0x%08x\n", + udma_cqe_aux_info_type_str[i], info->cqe_aux_info[i]); +} + +static void (*udma_cqe_aux_info_dump[14][2])(struct udma_dev *dev, + struct udma_cqe_aux_info_out *aux_info_out, + struct udma_cmd_query_cqe_aux_info *info) = { + {NULL, NULL}, + {dump_cqe_all_aux_info, dump_cqe_all_aux_info}, + {dump_cqe_server_loc_len_err_aux_info, + dump_cqe_client_loc_len_err_aux_info}, + {NULL, NULL}, + {dump_cqe_server_loc_access_err_aux_info, + dump_cqe_client_loc_access_err_aux_info}, + {dump_cqe_all_aux_info, + dump_cqe_client_rem_resp_len_err_aux_info}, + {dump_cqe_all_aux_info, dump_cqe_all_aux_info}, + {NULL, NULL}, + {dump_cqe_all_aux_info, + dump_cqe_client_rem_access_abort_err_aux_info}, + {dump_cqe_all_aux_info, + dump_cqe_client_ack_timeout_err_aux_info}, + {dump_cqe_all_aux_info, + dump_cqe_client_rnr_retry_cnt_exc_err_aux_info}, + {dump_cqe_all_aux_info, dump_cqe_all_aux_info}, + {NULL, NULL}, + {dump_cqe_all_aux_info, dump_cqe_all_aux_info}, +}; + +static void dump_fill_aux_info(struct udma_dev *dev, struct udma_ae_aux_info_out *aux_info_out, + struct udma_cmd_query_ae_aux_info *info, + enum udma_ae_aux_info_type *type, uint32_t aux_info_num) +{ + int i; + + if (aux_info_out->aux_info_type != NULL && + aux_info_out->aux_info_value != NULL && + aux_info_out->aux_info_num >= aux_info_num) { + for (i = 0; i < aux_info_num; i++) { + aux_info_out->aux_info_type[i] = type[i]; + aux_info_out->aux_info_value[i] = info->ae_aux_info[type[i]]; + } + aux_info_out->aux_info_num = aux_info_num; + } + + for (i = 0; i < aux_info_num; i++) + dev_info(dev->dev, "%s\t0x%08x\n", udma_ae_aux_info_type_str[type[i]], + info->ae_aux_info[type[i]]); +} + +static void dump_ae_tp_flush_done_aux_info(struct udma_dev *dev, + struct udma_ae_aux_info_out *aux_info_out, + struct udma_cmd_query_ae_aux_info *info) +{ + enum udma_ae_aux_info_type type[] = { + TP_RRP_FLUSH_TIMER_PKT_CNT, + TPP_DFX5, + }; + + uint32_t aux_info_num = ARRAY_SIZE(type); + + dump_fill_aux_info(dev, aux_info_out, info, type, aux_info_num); +} + +static void dump_ae_tp_err_aux_info(struct udma_dev *dev, + struct udma_ae_aux_info_out *aux_info_out, + struct udma_cmd_query_ae_aux_info *info) +{ + enum udma_ae_aux_info_type type[] = { + TWP_AE_DFX_FOR_AE, + TP_RRP_ERR_FLG_0_FOR_AE, + }; + uint32_t aux_info_num = ARRAY_SIZE(type); + + dump_fill_aux_info(dev, aux_info_out, info, type, aux_info_num); +} + +static void dump_ae_jetty_err_aux_info(struct udma_dev *dev, + struct udma_ae_aux_info_out *aux_info_out, + struct udma_cmd_query_ae_aux_info *info) +{ + enum udma_ae_aux_info_type type[] = { + TP_RRP_ERR_FLG_0_FOR_AE, + TP_RRP_ERR_FLG_1, + TP_RWP_INNER_ALM_FOR_AE, + TP_RCP_INNER_ALM_FOR_AE, + LQC_TA_TQEP_WQE_ERR, + LQC_TA_CQM_CQE_INNER_ALARM, + }; + uint32_t aux_info_num = ARRAY_SIZE(type); + + dump_fill_aux_info(dev, aux_info_out, info, type, aux_info_num); +} + +static void dump_ae_jfc_err_aux_info(struct udma_dev *dev, + struct udma_ae_aux_info_out *aux_info_out, + struct udma_cmd_query_ae_aux_info *info) +{ + enum udma_ae_aux_info_type type[] = { + LQC_TA_CQM_CQE_INNER_ALARM, + }; + uint32_t aux_info_num = ARRAY_SIZE(type); + + dump_fill_aux_info(dev, aux_info_out, info, type, aux_info_num); +} + +static void dump_ae_aux_info(struct udma_dev *dev, + struct udma_ae_aux_info_out *aux_info_out, + struct udma_cmd_query_ae_aux_info *info) +{ + switch (info->event_type) { + case UBASE_EVENT_TYPE_TP_FLUSH_DONE: + dump_ae_tp_flush_done_aux_info(dev, aux_info_out, info); + break; + case UBASE_EVENT_TYPE_TP_LEVEL_ERROR: + dump_ae_tp_err_aux_info(dev, aux_info_out, info); + break; + case UBASE_EVENT_TYPE_JETTY_LEVEL_ERROR: + if (info->sub_type == UBASE_SUBEVENT_TYPE_JFS_CHECK_ERROR) + dump_ae_jetty_err_aux_info(dev, aux_info_out, info); + else + dump_ae_jfc_err_aux_info(dev, aux_info_out, info); + break; + default: + break; + } +} + +static int send_cmd_query_cqe_aux_info(struct udma_dev *udma_dev, + struct udma_cmd_query_cqe_aux_info *info) +{ + struct ubase_cmd_buf cmd_in, cmd_out; + int ret; + + udma_fill_buf(&cmd_in, UDMA_CMD_GET_CQE_AUX_INFO, true, + sizeof(struct udma_cmd_query_cqe_aux_info), info); + udma_fill_buf(&cmd_out, UDMA_CMD_GET_CQE_AUX_INFO, true, + sizeof(struct udma_cmd_query_cqe_aux_info), info); + + ret = ubase_cmd_send_inout(udma_dev->comdev.adev, &cmd_in, &cmd_out); + if (ret) + dev_err(udma_dev->dev, + "failed to query cqe aux info, ret = %d.\n", ret); + + return ret; +} + +static void free_kernel_cqe_aux_info(struct udma_cqe_aux_info_out *user_aux_info_out, + struct udma_cqe_aux_info_out *aux_info_out) +{ + if (!user_aux_info_out->aux_info_type) + return; + + kfree(aux_info_out->aux_info_type); + aux_info_out->aux_info_type = NULL; + + kfree(aux_info_out->aux_info_value); + aux_info_out->aux_info_value = NULL; +} + +static int copy_out_cqe_data_from_user(struct udma_dev *udma_dev, + struct ubcore_user_ctl_out *out, + struct udma_cqe_aux_info_out *aux_info_out, + struct ubcore_ucontext *uctx, + struct udma_cqe_aux_info_out *user_aux_info_out) +{ + if (out->addr != 0 && out->len == sizeof(struct udma_cqe_aux_info_out)) { + memcpy(aux_info_out, (void *)(uintptr_t)out->addr, + sizeof(struct udma_cqe_aux_info_out)); + if (uctx && aux_info_out->aux_info_num > 0 && + aux_info_out->aux_info_type != NULL && + aux_info_out->aux_info_value != NULL) { + if (aux_info_out->aux_info_num > MAX_CQE_AUX_INFO_TYPE_NUM) { + dev_err(udma_dev->dev, + "invalid cqe aux info num %u.\n", + aux_info_out->aux_info_num); + return -EINVAL; + } + + user_aux_info_out->aux_info_type = aux_info_out->aux_info_type; + user_aux_info_out->aux_info_value = aux_info_out->aux_info_value; + aux_info_out->aux_info_type = + kcalloc(aux_info_out->aux_info_num, + sizeof(enum udma_cqe_aux_info_type), GFP_KERNEL); + if (!aux_info_out->aux_info_type) + return -ENOMEM; + + aux_info_out->aux_info_value = + kcalloc(aux_info_out->aux_info_num, + sizeof(uint32_t), GFP_KERNEL); + if (!aux_info_out->aux_info_value) { + kfree(aux_info_out->aux_info_type); + return -ENOMEM; + } + } + } + + return 0; +} + +static int copy_out_cqe_data_to_user(struct udma_dev *udma_dev, + struct ubcore_user_ctl_out *out, + struct udma_cqe_aux_info_out *aux_info_out, + struct ubcore_ucontext *uctx, + struct udma_cqe_aux_info_out *user_aux_info_out) +{ + unsigned long byte; + + if (out->addr != 0 && out->len == sizeof(struct udma_cqe_aux_info_out)) { + if (uctx && aux_info_out->aux_info_num > 0 && + aux_info_out->aux_info_type != NULL && + aux_info_out->aux_info_value != NULL) { + byte = copy_to_user((void __user *)user_aux_info_out->aux_info_type, + (void *)aux_info_out->aux_info_type, + aux_info_out->aux_info_num * + sizeof(enum udma_cqe_aux_info_type)); + if (byte) { + dev_err(udma_dev->dev, + "copy resp to aux info type failed, byte = %lu.\n", byte); + return -EFAULT; + } + + byte = copy_to_user((void __user *)user_aux_info_out->aux_info_value, + (void *)aux_info_out->aux_info_value, + aux_info_out->aux_info_num * + sizeof(uint32_t)); + if (byte) { + dev_err(udma_dev->dev, + "copy resp to aux info value failed, byte = %lu.\n", byte); + return -EFAULT; + } + + kfree(aux_info_out->aux_info_type); + kfree(aux_info_out->aux_info_value); + aux_info_out->aux_info_type = user_aux_info_out->aux_info_type; + aux_info_out->aux_info_value = user_aux_info_out->aux_info_value; + } + memcpy((void *)(uintptr_t)out->addr, aux_info_out, + sizeof(struct udma_cqe_aux_info_out)); + } + + return 0; +} + +int udma_query_cqe_aux_info(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out) +{ + struct udma_cqe_aux_info_out user_aux_info_out = {}; + struct udma_cqe_aux_info_out aux_info_out = {}; + struct udma_cmd_query_cqe_aux_info info = {}; + struct udma_cqe_info_in cqe_info_in = {}; + struct udma_dev *udev = to_udma_dev(dev); + int ret; + + if (udma_check_base_param(in->addr, in->len, sizeof(struct udma_cqe_info_in))) { + dev_err(udev->dev, "parameter invalid in query cqe aux info, in_len = %u.\n", + in->len); + return -EINVAL; + } + memcpy(&cqe_info_in, (void *)(uintptr_t)in->addr, + sizeof(struct udma_cqe_info_in)); + + info.status = cqe_info_in.status; + info.is_client = !(cqe_info_in.s_r & 1); + if (cqe_info_in.status >= ARRAY_SIZE(udma_cqe_aux_info_dump) || + udma_cqe_aux_info_dump[info.status][info.is_client] == NULL) { + dev_err(udev->dev, "status %u is invalid or does not need to be queried.\n", + cqe_info_in.status); + return -EINVAL; + } + + ret = copy_out_cqe_data_from_user(udev, out, &aux_info_out, uctx, &user_aux_info_out); + if (ret) { + dev_err(udev->dev, + "copy out data from user failed, ret = %d.\n", ret); + return ret; + } + + ret = send_cmd_query_cqe_aux_info(udev, &info); + if (ret) { + dev_err(udev->dev, + "send cmd query aux info failed, ret = %d.\n", + ret); + free_kernel_cqe_aux_info(&user_aux_info_out, &aux_info_out); + return ret; + } + + udma_cqe_aux_info_dump[info.status][info.is_client](udev, &aux_info_out, &info); + + ret = copy_out_cqe_data_to_user(udev, out, &aux_info_out, uctx, &user_aux_info_out); + if (ret) { + dev_err(udev->dev, + "copy out data to user failed, ret = %d.\n", ret); + free_kernel_cqe_aux_info(&user_aux_info_out, &aux_info_out); + } + + return ret; +} + +static int to_hw_ae_event_type(struct udma_dev *udma_dev, uint32_t event_type, + struct udma_cmd_query_ae_aux_info *info) +{ + switch (event_type) { + case UBCORE_EVENT_TP_FLUSH_DONE: + info->event_type = UBASE_EVENT_TYPE_TP_FLUSH_DONE; + break; + case UBCORE_EVENT_TP_ERR: + info->event_type = UBASE_EVENT_TYPE_TP_LEVEL_ERROR; + break; + case UBCORE_EVENT_JFS_ERR: + case UBCORE_EVENT_JETTY_ERR: + info->event_type = UBASE_EVENT_TYPE_JETTY_LEVEL_ERROR; + info->sub_type = UBASE_SUBEVENT_TYPE_JFS_CHECK_ERROR; + break; + case UBCORE_EVENT_JFC_ERR: + info->event_type = UBASE_EVENT_TYPE_JETTY_LEVEL_ERROR; + info->sub_type = UBASE_SUBEVENT_TYPE_JFC_CHECK_ERROR; + break; + default: + dev_err(udma_dev->dev, "Invalid event type %u.\n", event_type); + return -EINVAL; + } + + return 0; +} + +static int send_cmd_query_ae_aux_info(struct udma_dev *udma_dev, + struct udma_cmd_query_ae_aux_info *info) +{ + struct ubase_cmd_buf cmd_in, cmd_out; + int ret; + + udma_fill_buf(&cmd_in, UDMA_CMD_GET_AE_AUX_INFO, true, + sizeof(struct udma_cmd_query_ae_aux_info), info); + udma_fill_buf(&cmd_out, UDMA_CMD_GET_AE_AUX_INFO, true, + sizeof(struct udma_cmd_query_ae_aux_info), info); + + ret = ubase_cmd_send_inout(udma_dev->comdev.adev, &cmd_in, &cmd_out); + if (ret) + dev_err(udma_dev->dev, + "failed to query ae aux info, ret = %d.\n", ret); + + return ret; +} + +static void free_kernel_ae_aux_info(struct udma_ae_aux_info_out *user_aux_info_out, + struct udma_ae_aux_info_out *aux_info_out) +{ + if (!user_aux_info_out->aux_info_type) + return; + + kfree(aux_info_out->aux_info_type); + aux_info_out->aux_info_type = NULL; + + kfree(aux_info_out->aux_info_value); + aux_info_out->aux_info_value = NULL; +} + +static int copy_out_ae_data_from_user(struct udma_dev *udma_dev, + struct ubcore_user_ctl_out *out, + struct udma_ae_aux_info_out *aux_info_out, + struct ubcore_ucontext *uctx, + struct udma_ae_aux_info_out *user_aux_info_out) +{ + if (out->addr != 0 && out->len == sizeof(struct udma_ae_aux_info_out)) { + memcpy(aux_info_out, (void *)(uintptr_t)out->addr, + sizeof(struct udma_ae_aux_info_out)); + if (uctx && aux_info_out->aux_info_num > 0 && + aux_info_out->aux_info_type != NULL && + aux_info_out->aux_info_value != NULL) { + if (aux_info_out->aux_info_num > MAX_AE_AUX_INFO_TYPE_NUM) { + dev_err(udma_dev->dev, + "invalid ae aux info num %u.\n", + aux_info_out->aux_info_num); + return -EINVAL; + } + + user_aux_info_out->aux_info_type = aux_info_out->aux_info_type; + user_aux_info_out->aux_info_value = aux_info_out->aux_info_value; + aux_info_out->aux_info_type = + kcalloc(aux_info_out->aux_info_num, + sizeof(enum udma_ae_aux_info_type), GFP_KERNEL); + if (!aux_info_out->aux_info_type) + return -ENOMEM; + + aux_info_out->aux_info_value = + kcalloc(aux_info_out->aux_info_num, + sizeof(uint32_t), GFP_KERNEL); + if (!aux_info_out->aux_info_value) { + kfree(aux_info_out->aux_info_type); + return -ENOMEM; + } + } + } + + return 0; +} + +static int copy_out_ae_data_to_user(struct udma_dev *udma_dev, + struct ubcore_user_ctl_out *out, + struct udma_ae_aux_info_out *aux_info_out, + struct ubcore_ucontext *uctx, + struct udma_ae_aux_info_out *user_aux_info_out) +{ + unsigned long byte; + + if (out->addr != 0 && out->len == sizeof(struct udma_ae_aux_info_out)) { + if (uctx && aux_info_out->aux_info_num > 0 && + aux_info_out->aux_info_type != NULL && + aux_info_out->aux_info_value != NULL) { + byte = copy_to_user((void __user *)user_aux_info_out->aux_info_type, + (void *)aux_info_out->aux_info_type, + aux_info_out->aux_info_num * + sizeof(enum udma_ae_aux_info_type)); + if (byte) { + dev_err(udma_dev->dev, + "copy resp to aux info type failed, byte = %lu.\n", byte); + return -EFAULT; + } + + byte = copy_to_user((void __user *)user_aux_info_out->aux_info_value, + (void *)aux_info_out->aux_info_value, + aux_info_out->aux_info_num * + sizeof(uint32_t)); + if (byte) { + dev_err(udma_dev->dev, + "copy resp to aux info value failed, byte = %lu.\n", byte); + return -EFAULT; + } + + kfree(aux_info_out->aux_info_type); + kfree(aux_info_out->aux_info_value); + aux_info_out->aux_info_type = user_aux_info_out->aux_info_type; + aux_info_out->aux_info_value = user_aux_info_out->aux_info_value; + } + memcpy((void *)(uintptr_t)out->addr, aux_info_out, + sizeof(struct udma_ae_aux_info_out)); + } + + return 0; +} + +int udma_query_ae_aux_info(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out) +{ + struct udma_ae_aux_info_out user_aux_info_out = {}; + struct udma_ae_aux_info_out aux_info_out = {}; + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_cmd_query_ae_aux_info info = {}; + struct udma_ae_info_in ae_info_in = {}; + int ret; + + if (udma_check_base_param(in->addr, in->len, sizeof(struct udma_ae_info_in))) { + dev_err(udma_dev->dev, "parameter invalid in query ae aux info, in_len = %u.\n", + in->len); + return -EINVAL; + } + memcpy(&ae_info_in, (void *)(uintptr_t)in->addr, + sizeof(struct udma_ae_info_in)); + ret = to_hw_ae_event_type(udma_dev, ae_info_in.event_type, &info); + if (ret) + return ret; + + ret = copy_out_ae_data_from_user(udma_dev, out, &aux_info_out, uctx, &user_aux_info_out); + if (ret) { + dev_err(udma_dev->dev, + "copy out data from user failed, ret = %d.\n", ret); + return ret; + } + + ret = send_cmd_query_ae_aux_info(udma_dev, &info); + if (ret) { + dev_err(udma_dev->dev, + "send cmd query aux info failed, ret = %d.\n", + ret); + free_kernel_ae_aux_info(&user_aux_info_out, &aux_info_out); + return ret; + } + + dump_ae_aux_info(udma_dev, &aux_info_out, &info); + + ret = copy_out_ae_data_to_user(udma_dev, out, &aux_info_out, uctx, &user_aux_info_out); + if (ret) { + dev_err(udma_dev->dev, + "copy out data to user failed, ret = %d.\n", ret); + free_kernel_ae_aux_info(&user_aux_info_out, &aux_info_out); + } + + return ret; +} + +static udma_user_ctl_ops g_udma_user_ctl_k_ops[] = { + [UDMA_USER_CTL_CREATE_JFS_EX] = udma_create_jfs_ops_ex, + [UDMA_USER_CTL_DELETE_JFS_EX] = udma_delete_jfs_ops_ex, + [UDMA_USER_CTL_CREATE_JFC_EX] = udma_create_jfc_ops_ex, + [UDMA_USER_CTL_DELETE_JFC_EX] = udma_delete_jfc_ops_ex, + [UDMA_USER_CTL_SET_CQE_ADDR] = udma_set_cqe_ex, + [UDMA_USER_CTL_QUERY_UE_INFO] = udma_query_ue_info_ex, + [UDMA_USER_CTL_GET_DEV_RES_RATIO] = udma_get_dev_resource_ratio, + [UDMA_USER_CTL_NPU_REGISTER_INFO_CB] = udma_register_npu_cb, + [UDMA_USER_CTL_NPU_UNREGISTER_INFO_CB] = udma_unregister_npu_cb, + [UDMA_USER_CTL_QUERY_TP_SPORT] = udma_ctrlq_query_tp_sport, + [UDMA_USER_CTL_QUERY_CQE_AUX_INFO] = udma_query_cqe_aux_info, + [UDMA_USER_CTL_QUERY_AE_AUX_INFO] = udma_query_ae_aux_info, + [UDMA_USER_CTL_QUERY_UBMEM_INFO] = udma_ctrlq_query_ubmem_info, + [UDMA_USER_CTL_QUERY_PAIR_DEVNUM] = udma_query_pair_dev_count, +}; + +static udma_user_ctl_ops g_udma_user_ctl_u_ops[] = { + [UDMA_USER_CTL_CREATE_JFS_EX] = NULL, + [UDMA_USER_CTL_DELETE_JFS_EX] = NULL, + [UDMA_USER_CTL_CREATE_JFC_EX] = NULL, + [UDMA_USER_CTL_DELETE_JFC_EX] = NULL, + [UDMA_USER_CTL_SET_CQE_ADDR] = NULL, + [UDMA_USER_CTL_QUERY_UE_INFO] = NULL, + [UDMA_USER_CTL_GET_DEV_RES_RATIO] = NULL, + [UDMA_USER_CTL_NPU_REGISTER_INFO_CB] = NULL, + [UDMA_USER_CTL_NPU_UNREGISTER_INFO_CB] = NULL, + [UDMA_USER_CTL_QUERY_TP_SPORT] = udma_ctrlq_query_tp_sport, + [UDMA_USER_CTL_QUERY_CQE_AUX_INFO] = udma_query_cqe_aux_info, + [UDMA_USER_CTL_QUERY_AE_AUX_INFO] = udma_query_ae_aux_info, + [UDMA_USER_CTL_QUERY_UBMEM_INFO] = NULL, + [UDMA_USER_CTL_QUERY_PAIR_DEVNUM] = NULL, +}; + +static int udma_user_data(struct ubcore_device *dev, + struct ubcore_user_ctl *k_user_ctl) +{ + struct udma_dev *udev = to_udma_dev(dev); + struct ubcore_user_ctl_out out = {}; + struct ubcore_user_ctl_in in = {}; + unsigned long byte; + int ret; + + if (k_user_ctl->in.len >= UDMA_HW_PAGE_SIZE || k_user_ctl->out.len >= UDMA_HW_PAGE_SIZE) { + dev_err(udev->dev, "The len exceeds the maximum value in user ctrl.\n"); + return -EINVAL; + } + + in.opcode = k_user_ctl->in.opcode; + if (!g_udma_user_ctl_u_ops[in.opcode]) { + dev_err(udev->dev, "invalid user opcode: 0x%x.\n", in.opcode); + return -EINVAL; + } + + if (k_user_ctl->in.len) { + in.addr = (uint64_t)kzalloc(k_user_ctl->in.len, GFP_KERNEL); + if (!in.addr) + return -ENOMEM; + + in.len = k_user_ctl->in.len; + byte = copy_from_user((void *)(uintptr_t)in.addr, + (void __user *)(uintptr_t)k_user_ctl->in.addr, + k_user_ctl->in.len); + if (byte) { + dev_err(udev->dev, + "failed to copy user data in user ctrl, byte = %lu.\n", byte); + kfree((void *)in.addr); + return -EFAULT; + } + } + + if (k_user_ctl->out.len) { + out.addr = (uint64_t)kzalloc(k_user_ctl->out.len, GFP_KERNEL); + if (!out.addr) { + kfree((void *)in.addr); + + return -ENOMEM; + } + out.len = k_user_ctl->out.len; + + if (k_user_ctl->out.addr) { + byte = copy_from_user((void *)(uintptr_t)out.addr, + (void __user *)(uintptr_t)k_user_ctl->out.addr, + k_user_ctl->out.len); + if (byte) { + dev_err(udev->dev, + "failed to copy user data out user ctrl, byte = %lu.\n", + byte); + kfree((void *)out.addr); + kfree((void *)in.addr); + + return -EFAULT; + } + } + } + + ret = g_udma_user_ctl_u_ops[in.opcode](dev, k_user_ctl->uctx, &in, &out); + kfree((void *)in.addr); + + if (out.addr) { + byte = copy_to_user((void __user *)(uintptr_t)k_user_ctl->out.addr, + (void *)(uintptr_t)out.addr, min(out.len, k_user_ctl->out.len)); + if (byte) { + dev_err(udev->dev, + "copy resp to user failed in user ctrl, byte = %lu.\n", byte); + ret = -EFAULT; + } + + kfree((void *)out.addr); + } + + return ret; +} + +int udma_user_ctl(struct ubcore_device *dev, struct ubcore_user_ctl *k_user_ctl) +{ + struct udma_dev *udev; + + if (dev == NULL || k_user_ctl == NULL) + return -EINVAL; + + udev = to_udma_dev(dev); + + if (k_user_ctl->in.opcode >= UDMA_USER_CTL_MAX) { + dev_err(udev->dev, "invalid opcode: 0x%x.\n", k_user_ctl->in.opcode); + return -EINVAL; + } + + if (k_user_ctl->uctx) + return udma_user_data(dev, k_user_ctl); + + if (!g_udma_user_ctl_k_ops[k_user_ctl->in.opcode]) { + dev_err(udev->dev, "invalid user opcode: 0x%x.\n", k_user_ctl->in.opcode); + return -EINVAL; + } + + return g_udma_user_ctl_k_ops[k_user_ctl->in.opcode](dev, k_user_ctl->uctx, &k_user_ctl->in, + &k_user_ctl->out); +} diff --git a/drivers/ub/urma/hw/udma/udma_ctl.h b/drivers/ub/urma/hw/udma/udma_ctl.h new file mode 100644 index 0000000000000000000000000000000000000000..da1d082aeec4df7eb45220c7dd868238d2489c35 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_ctl.h @@ -0,0 +1,213 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef _UB_UMDK_URMA_UDMA_UDMA_CTL_H_ +#define _UB_UMDK_URMA_UDMA_UDMA_CTL_H_ + +#include + +#define UDMA_BUS_INSTANCE_SEID_SIZE 4 +#define UDMA_EID_PAIRS_COUNT 8 + +union udma_k_jfs_flag { + struct { + uint32_t sq_cstm : 1; + uint32_t db_cstm : 1; + uint32_t db_ctl_cstm : 1; + uint32_t reserved : 29; + } bs; + uint32_t value; +}; + +struct udma_que_cfg_ex { + uint32_t buff_size; + void *buff; +}; + +struct udma_jfs_cstm_cfg { + struct udma_que_cfg_ex sq; /* PA; should be converted by phys_to_virt. */ +}; + +struct udma_jfs_cfg_ex { + struct ubcore_jfs_cfg base_cfg; + struct ubcore_udata udata; + struct udma_jfs_cstm_cfg cstm_cfg; + ubcore_event_callback_t jfae_handler; +}; + +struct udma_jfc_cstm_cfg { + struct udma_que_cfg_ex cq; /* PA; should be using stars hw register addr. */ +}; + +struct udma_jfc_cfg_ex { + struct ubcore_jfc_cfg base_cfg; + struct ubcore_udata udata; + struct udma_jfc_cstm_cfg cstm_cfg; + ubcore_comp_callback_t jfce_handler; + ubcore_event_callback_t jfae_handler; +}; + +enum udma_jfc_type { + UDMA_NORMAL_JFC_TYPE, + UDMA_STARS_JFC_TYPE, + UDMA_CCU_JFC_TYPE, + UDMA_KERNEL_STARS_JFC_TYPE, + UDMA_JFC_TYPE_NUM, +}; + +struct udma_set_cqe_ex { + uint64_t addr; + uint32_t len; + enum udma_jfc_type jfc_type; +}; + +struct udma_ue_info_ex { + uint16_t ue_id; + uint32_t chip_id; + uint32_t die_id; + uint32_t offset_len; + resource_size_t db_base_addr; + resource_size_t dwqe_addr; + resource_size_t register_base_addr; +}; + +struct udma_tp_sport_in { + uint32_t tpn; +}; + +struct udma_tp_sport_out { + uint32_t data_udp_srcport; + uint32_t ack_udp_srcport; +}; + +struct udma_cqe_info_in { + enum ubcore_cr_status status; + uint8_t s_r; +}; + +enum udma_cqe_aux_info_type { + TPP2TQEM_WR_CNT, + DEVICE_RAS_STATUS_2, + RXDMA_WR_PAYL_AXI_ERR, + RXDMA_HEAD_SPLIT_ERR_FLAG0, + RXDMA_HEAD_SPLIT_ERR_FLAG1, + RXDMA_HEAD_SPLIT_ERR_FLAG2, + RXDMA_HEAD_SPLIT_ERR_FLAG3, + TP_RCP_INNER_ALM_FOR_CQE, + TWP_AE_DFX_FOR_CQE, + PA_OUT_PKT_ERR_CNT, + TP_DAM_AXI_ALARM, + TP_DAM_VFT_BT_ALARM, + TP_EUM_AXI_ALARM, + TP_EUM_VFT_BT_ALARM, + TP_TPMM_AXI_ALARM, + TP_TPMM_VFT_BT_ALARM, + TP_TPGCM_AXI_ALARM, + TP_TPGCM_VFT_BT_ALARM, + TWP_ALM, + TP_RWP_INNER_ALM_FOR_CQE, + TWP_DFX21, + LQC_TA_RNR_TANACK_CNT, + FVT, + RQMT0, + RQMT1, + RQMT2, + RQMT3, + RQMT4, + RQMT5, + RQMT6, + RQMT7, + RQMT8, + RQMT9, + RQMT10, + RQMT11, + RQMT12, + RQMT13, + RQMT14, + RQMT15, + PROC_ERROR_ALM, + LQC_TA_TIMEOUT_TAACK_CNT, + TP_RRP_ERR_FLG_0_FOR_CQE, + MAX_CQE_AUX_INFO_TYPE_NUM +}; + +struct udma_cqe_aux_info_out { + enum udma_cqe_aux_info_type *aux_info_type; + uint32_t *aux_info_value; + uint32_t aux_info_num; +}; + +struct udma_ae_info_in { + uint32_t event_type; +}; + +enum udma_ae_aux_info_type { + TP_RRP_FLUSH_TIMER_PKT_CNT, + TPP_DFX5, + TWP_AE_DFX_FOR_AE, + TP_RRP_ERR_FLG_0_FOR_AE, + TP_RRP_ERR_FLG_1, + TP_RWP_INNER_ALM_FOR_AE, + TP_RCP_INNER_ALM_FOR_AE, + LQC_TA_TQEP_WQE_ERR, + LQC_TA_CQM_CQE_INNER_ALARM, + MAX_AE_AUX_INFO_TYPE_NUM +}; + +struct udma_ae_aux_info_out { + enum udma_ae_aux_info_type *aux_info_type; + uint32_t *aux_info_value; + uint32_t aux_info_num; +}; + +enum udma_user_ctl_opcode { + UDMA_USER_CTL_CREATE_JFS_EX, + UDMA_USER_CTL_DELETE_JFS_EX, + UDMA_USER_CTL_CREATE_JFC_EX, + UDMA_USER_CTL_DELETE_JFC_EX, + UDMA_USER_CTL_SET_CQE_ADDR, + UDMA_USER_CTL_QUERY_UE_INFO, + UDMA_USER_CTL_GET_DEV_RES_RATIO, + UDMA_USER_CTL_NPU_REGISTER_INFO_CB, + UDMA_USER_CTL_NPU_UNREGISTER_INFO_CB, + UDMA_USER_CTL_QUERY_TP_SPORT, + UDMA_USER_CTL_QUERY_CQE_AUX_INFO, + UDMA_USER_CTL_QUERY_AE_AUX_INFO, + UDMA_USER_CTL_QUERY_UBMEM_INFO, + UDMA_USER_CTL_QUERY_PAIR_DEVNUM, + UDMA_USER_CTL_MAX, +}; + +struct udma_ctrlq_event_nb { + uint8_t opcode; + int (*crq_handler)(struct ubcore_device *dev, void *data, uint16_t len); +}; + +struct udma_dev_pair_info { + uint32_t peer_dev_id; + uint32_t slot_id; + uint32_t pair_num; + struct { + uint32_t local_eid[UDMA_BUS_INSTANCE_SEID_SIZE]; + uint32_t remote_eid[UDMA_BUS_INSTANCE_SEID_SIZE]; + uint32_t flag : 16; + uint32_t hop : 4; + uint32_t rsv : 12; + } eid_pairs[UDMA_EID_PAIRS_COUNT]; +}; + +static inline bool udma_check_base_param(uint64_t addr, uint32_t in_len, uint32_t len) +{ + return (addr == 0 || in_len != len); +} + +typedef int (*udma_user_ctl_ops)(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out); + +int udma_user_ctl(struct ubcore_device *dev, struct ubcore_user_ctl *k_user_ctl); +int udma_query_cqe_aux_info(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out); +int udma_query_ae_aux_info(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out); + +#endif /* _UB_UMDK_URMA_UDMA_UDMA_CTL_H_ */ diff --git a/drivers/ub/urma/hw/udma/udma_ctrlq_tp.c b/drivers/ub/urma/hw/udma/udma_ctrlq_tp.c new file mode 100644 index 0000000000000000000000000000000000000000..86d68ace700085c210e9e247b277581a64fb40ea --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_ctrlq_tp.c @@ -0,0 +1,840 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt + +#include +#include +#include +#include +#include "udma_cmd.h" +#include +#include "udma_common.h" +#include "udma_ctrlq_tp.h" + +static void udma_ctrlq_set_tp_msg(struct ubase_ctrlq_msg *msg, void *in, + uint16_t in_len, void *out, uint16_t out_len) +{ + msg->service_ver = UBASE_CTRLQ_SER_VER_01; + msg->service_type = UBASE_CTRLQ_SER_TYPE_TP_ACL; + msg->need_resp = 1; + msg->is_resp = 0; + msg->in_size = in_len; + msg->in = in; + msg->out_size = out_len; + msg->out = out; +} + +int udma_ctrlq_remove_single_tp(struct udma_dev *udev, uint32_t tpn, int status) +{ + struct udma_ctrlq_remove_single_tp_req_data tp_cfg_req = {}; + struct ubase_ctrlq_msg msg = {}; + int r_status = 0; + int ret; + + tp_cfg_req.tpn = tpn; + tp_cfg_req.tp_status = (uint32_t)status; + msg.opcode = UDMA_CMD_CTRLQ_REMOVE_SINGLE_TP; + udma_ctrlq_set_tp_msg(&msg, (void *)&tp_cfg_req, + sizeof(tp_cfg_req), &r_status, sizeof(int)); + + ret = ubase_ctrlq_send_msg(udev->comdev.adev, &msg); + if (ret) + dev_err(udev->dev, "remove single tp %u failed, ret %d status %d.\n", + tpn, ret, r_status); + + return ret; +} + +static int udma_send_req_to_ue(struct udma_dev *udma_dev, uint8_t ue_idx) +{ + struct ubcore_resp *ubcore_req; + int ret; + + ubcore_req = kzalloc(sizeof(*ubcore_req), GFP_KERNEL); + if (!ubcore_req) + return -ENOMEM; + + ret = send_resp_to_ue(udma_dev, ubcore_req, ue_idx, + UDMA_CMD_NOTIFY_UE_FLUSH_DONE); + if (ret) + dev_err(udma_dev->dev, "fail to notify ue the tp flush done, ret %d.\n", ret); + + kfree(ubcore_req); + + return ret; +} + +static struct udma_ue_idx_table *udma_find_ue_idx_by_tpn(struct udma_dev *udev, + uint32_t tpn) +{ + struct udma_ue_idx_table *tp_ue_idx_info; + + xa_lock(&udev->tpn_ue_idx_table); + tp_ue_idx_info = xa_load(&udev->tpn_ue_idx_table, tpn); + if (!tp_ue_idx_info) { + dev_warn(udev->dev, "ue idx info not exist, tpn %u.\n", tpn); + xa_unlock(&udev->tpn_ue_idx_table); + + return NULL; + } + + __xa_erase(&udev->tpn_ue_idx_table, tpn); + xa_unlock(&udev->tpn_ue_idx_table); + + return tp_ue_idx_info; +} + +int udma_ctrlq_tp_flush_done(struct udma_dev *udev, uint32_t tpn) +{ + struct udma_ctrlq_tp_flush_done_req_data tp_cfg_req = {}; + struct udma_ue_idx_table *tp_ue_idx_info; + struct ubase_ctrlq_msg msg = {}; + int ret = 0; + uint32_t i; + + tp_ue_idx_info = udma_find_ue_idx_by_tpn(udev, tpn); + if (tp_ue_idx_info) { + for (i = 0; i < tp_ue_idx_info->num; i++) + (void)udma_send_req_to_ue(udev, tp_ue_idx_info->ue_idx[i]); + + kfree(tp_ue_idx_info); + } else { + ret = udma_open_ue_rx(udev, true, false, false, 0); + if (ret) + dev_err(udev->dev, "udma open ue rx failed in tp flush done.\n"); + } + + tp_cfg_req.tpn = tpn; + msg.opcode = UDMA_CMD_CTRLQ_TP_FLUSH_DONE; + udma_ctrlq_set_tp_msg(&msg, (void *)&tp_cfg_req, sizeof(tp_cfg_req), NULL, 0); + ret = ubase_ctrlq_send_msg(udev->comdev.adev, &msg); + if (ret) + dev_err(udev->dev, "tp flush done ctrlq tp %u failed, ret %d.\n", tpn, ret); + + return ret; +} + +int udma_get_dev_resource_ratio(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out) +{ + struct udma_dev_resource_ratio dev_res = {}; + struct udma_dev_pair_info dev_res_out = {}; + struct udma_dev *udev = to_udma_dev(dev); + struct ubase_ctrlq_msg ctrlq_msg = {}; + int ret = 0; + + if (udma_check_base_param(in->addr, in->len, sizeof(dev_res.index))) { + dev_err(udev->dev, "parameter invalid in get dev res, len = %u.\n", in->len); + return -EINVAL; + } + + if (out->addr == 0 || out->len != sizeof(dev_res_out)) { + dev_err(udev->dev, "get dev resource ratio, addr is NULL:%d, len:%u.\n", + out->addr == 0, out->len); + return -EINVAL; + } + + memcpy(&dev_res.index, (void *)(uintptr_t)in->addr, sizeof(dev_res.index)); + + ret = ubase_get_bus_eid(udev->comdev.adev, &dev_res.eid); + if (ret) { + dev_err(udev->dev, "get dev bus eid failed, ret is %d.\n", ret); + return ret; + } + + ctrlq_msg.service_type = UBASE_CTRLQ_SER_TYPE_DEV_REGISTER; + ctrlq_msg.service_ver = UBASE_CTRLQ_SER_VER_01; + ctrlq_msg.need_resp = 1; + ctrlq_msg.in_size = sizeof(dev_res); + ctrlq_msg.in = (void *)&dev_res; + ctrlq_msg.out_size = sizeof(dev_res_out); + ctrlq_msg.out = &dev_res_out; + ctrlq_msg.opcode = UDMA_CTRLQ_GET_DEV_RESOURCE_RATIO; + + ret = ubase_ctrlq_send_msg(udev->comdev.adev, &ctrlq_msg); + if (ret) { + dev_err(udev->dev, "get dev res send ctrlq msg failed, ret is %d.\n", ret); + return ret; + } + memcpy((void *)(uintptr_t)out->addr, &dev_res_out, sizeof(dev_res_out)); + + return ret; +} + +static int udma_dev_res_ratio_ctrlq_handler(struct auxiliary_device *adev, + uint8_t service_ver, void *data, + uint16_t len, uint16_t seq) +{ + struct udma_dev *udev = (struct udma_dev *)get_udma_dev(adev); + struct udma_ctrlq_event_nb *udma_cb; + int ret; + + if (service_ver != UBASE_CTRLQ_SER_VER_01) { + dev_err(udev->dev, "Unsupported server version (%u).\n", service_ver); + return -EOPNOTSUPP; + } + + mutex_lock(&udev->npu_nb_mutex); + udma_cb = xa_load(&udev->npu_nb_table, UDMA_CTRLQ_NOTIFY_DEV_RESOURCE_RATIO); + if (!udma_cb) { + dev_err(udev->dev, "failed to query npu info cb while xa_load.\n"); + mutex_unlock(&udev->npu_nb_mutex); + return -EINVAL; + } + + ret = udma_cb->crq_handler(&udev->ub_dev, data, len); + if (ret) + dev_err(udev->dev, "npu crq handler failed, ret = %d.\n", ret); + mutex_unlock(&udev->npu_nb_mutex); + + return ret; +} + +int udma_register_npu_cb(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out) +{ + struct ubase_ctrlq_event_nb ubase_cb = {}; + struct udma_dev *udev = to_udma_dev(dev); + struct udma_ctrlq_event_nb *udma_cb; + int ret; + + if (udma_check_base_param(in->addr, in->len, sizeof(udma_cb->crq_handler))) { + dev_err(udev->dev, "parameter invalid in register npu cb, len = %u.\n", in->len); + return -EINVAL; + } + + udma_cb = kzalloc(sizeof(*udma_cb), GFP_KERNEL); + if (!udma_cb) + return -ENOMEM; + + udma_cb->opcode = UDMA_CTRLQ_NOTIFY_DEV_RESOURCE_RATIO; + udma_cb->crq_handler = (void *)(uintptr_t)in->addr; + + mutex_lock(&udev->npu_nb_mutex); + if (xa_load(&udev->npu_nb_table, UDMA_CTRLQ_NOTIFY_DEV_RESOURCE_RATIO)) { + dev_err(udev->dev, "query npu info callback exist.\n"); + ret = -EINVAL; + goto err_release_udma_cb; + } + ret = xa_err(__xa_store(&udev->npu_nb_table, udma_cb->opcode, udma_cb, GFP_KERNEL)); + if (ret) { + dev_err(udev->dev, + "save crq nb entry failed, opcode is %u, ret is %d.\n", + udma_cb->opcode, ret); + goto err_release_udma_cb; + } + + ubase_cb.service_type = UBASE_CTRLQ_SER_TYPE_DEV_REGISTER; + ubase_cb.opcode = UDMA_CTRLQ_NOTIFY_DEV_RESOURCE_RATIO; + ubase_cb.back = udev->comdev.adev; + ubase_cb.crq_handler = udma_dev_res_ratio_ctrlq_handler; + ret = ubase_ctrlq_register_crq_event(udev->comdev.adev, &ubase_cb); + if (ret) { + __xa_erase(&udev->npu_nb_table, UDMA_CTRLQ_NOTIFY_DEV_RESOURCE_RATIO); + dev_err(udev->dev, "ubase register npu crq event failed, ret is %d.\n", ret); + goto err_release_udma_cb; + } + mutex_unlock(&udev->npu_nb_mutex); + + return 0; + +err_release_udma_cb: + mutex_unlock(&udev->npu_nb_mutex); + kfree(udma_cb); + return ret; +} + +int udma_unregister_npu_cb(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out) +{ + struct udma_dev *udev = to_udma_dev(dev); + struct udma_ctrlq_event_nb *nb; + + ubase_ctrlq_unregister_crq_event(udev->comdev.adev, + UBASE_CTRLQ_SER_TYPE_DEV_REGISTER, + UDMA_CTRLQ_NOTIFY_DEV_RESOURCE_RATIO); + + mutex_lock(&udev->npu_nb_mutex); + nb = xa_load(&udev->npu_nb_table, UDMA_CTRLQ_NOTIFY_DEV_RESOURCE_RATIO); + if (!nb) { + dev_warn(udev->dev, "query npu info cb not exist.\n"); + goto err_find_npu_nb; + } + + __xa_erase(&udev->npu_nb_table, UDMA_CTRLQ_NOTIFY_DEV_RESOURCE_RATIO); + kfree(nb); + nb = NULL; + +err_find_npu_nb: + mutex_unlock(&udev->npu_nb_mutex); + return 0; +} + +static int udma_ctrlq_get_trans_type(struct udma_dev *dev, + enum ubcore_transport_mode trans_mode, + enum udma_ctrlq_trans_type *tp_type) +{ +#define UDMA_TRANS_MODE_NUM 5 + +struct udma_ctrlq_trans_map { + bool is_valid; + enum udma_ctrlq_trans_type tp_type; +}; + static struct udma_ctrlq_trans_map ctrlq_trans_map[UDMA_TRANS_MODE_NUM] = { + {false, UDMA_CTRLQ_TRANS_TYPE_MAX}, + {true, UDMA_CTRLQ_TRANS_TYPE_TP_RM}, + {true, UDMA_CTRLQ_TRANS_TYPE_TP_RC}, + {false, UDMA_CTRLQ_TRANS_TYPE_MAX}, + {true, UDMA_CTRLQ_TRANS_TYPE_TP_UM}, + }; + uint8_t transport_mode = (uint8_t)trans_mode; + + if ((transport_mode < UDMA_TRANS_MODE_NUM) && + ctrlq_trans_map[transport_mode].is_valid) { + *tp_type = ctrlq_trans_map[transport_mode].tp_type; + return 0; + } + + dev_err(dev->dev, "the trans_mode %u is not support.\n", trans_mode); + + return -EINVAL; +} + +static int udma_send_req_to_mue(struct udma_dev *dev, union ubcore_tp_handle *tp_handle) +{ + uint32_t data_len = (uint32_t)sizeof(struct udma_ue_tp_info); + struct udma_ue_tp_info *data; + struct ubcore_req *req_msg; + int ret; + + req_msg = kzalloc(sizeof(*req_msg) + data_len, GFP_KERNEL); + if (!req_msg) + return -ENOMEM; + + data = (struct udma_ue_tp_info *)req_msg->data; + data->start_tpn = tp_handle->bs.tpn_start; + data->tp_cnt = tp_handle->bs.tp_cnt; + req_msg->len = data_len; + ret = send_req_to_mue(dev, req_msg, UDMA_CMD_NOTIFY_MUE_SAVE_TP); + if (ret) + dev_err(dev->dev, "fail to notify mue save tp, ret %d.\n", ret); + + kfree(req_msg); + + return ret; +} + +static int udma_ctrlq_store_one_tpid(struct udma_dev *udev, struct xarray *ctrlq_tpid_table, + struct udma_ctrlq_tpid *tpid) +{ + struct udma_ctrlq_tpid *tpid_entity; + int ret; + + if (debug_switch) + dev_info(udev->dev, "udma ctrlq store one tpid start. tpid %u\n", tpid->tpid); + + if (xa_load(ctrlq_tpid_table, tpid->tpid)) { + dev_warn(udev->dev, + "the tpid already exists in ctrlq tpid table, tpid = %u.\n", + tpid->tpid); + return 0; + } + + tpid_entity = kzalloc(sizeof(*tpid_entity), GFP_KERNEL); + if (!tpid_entity) + return -ENOMEM; + + memcpy(tpid_entity, tpid, sizeof(*tpid)); + + ret = xa_err(xa_store(ctrlq_tpid_table, tpid->tpid, tpid_entity, GFP_KERNEL)); + if (ret) { + dev_err(udev->dev, + "store tpid entity failed, ret = %d, tpid = %u.\n", + ret, tpid->tpid); + kfree(tpid_entity); + } + + return ret; +} + +static void udma_ctrlq_erase_one_tpid(struct xarray *ctrlq_tpid_table, + uint32_t tpid) +{ + struct udma_ctrlq_tpid *tpid_entity; + + xa_lock(ctrlq_tpid_table); + tpid_entity = xa_load(ctrlq_tpid_table, tpid); + if (!tpid_entity) { + xa_unlock(ctrlq_tpid_table); + return; + } + __xa_erase(ctrlq_tpid_table, tpid); + kfree(tpid_entity); + xa_unlock(ctrlq_tpid_table); +} + +static int udma_ctrlq_get_tpid_list(struct udma_dev *udev, + struct udma_ctrlq_get_tp_list_req_data *tp_cfg_req, + struct ubcore_get_tp_cfg *tpid_cfg, + struct udma_ctrlq_tpid_list_rsp *tpid_list_resp) +{ + enum udma_ctrlq_trans_type trans_type; + struct ubase_ctrlq_msg msg = {}; + int ret; + + if (!tpid_cfg->flag.bs.ctp) { + if (udma_ctrlq_get_trans_type(udev, tpid_cfg->trans_mode, &trans_type) != 0) { + dev_err(udev->dev, "udma get ctrlq trans_type failed, trans_mode = %d.\n", + tpid_cfg->trans_mode); + return -EINVAL; + } + + tp_cfg_req->trans_type = (uint32_t)trans_type; + } else { + tp_cfg_req->trans_type = UDMA_CTRLQ_TRANS_TYPE_CTP; + } + + udma_swap_endian(tpid_cfg->local_eid.raw, tp_cfg_req->seid, + UDMA_EID_SIZE); + udma_swap_endian(tpid_cfg->peer_eid.raw, tp_cfg_req->deid, + UDMA_EID_SIZE); + + udma_ctrlq_set_tp_msg(&msg, (void *)tp_cfg_req, sizeof(*tp_cfg_req), + (void *)tpid_list_resp, sizeof(*tpid_list_resp)); + msg.opcode = UDMA_CMD_CTRLQ_GET_TP_LIST; + + ret = ubase_ctrlq_send_msg(udev->comdev.adev, &msg); + if (ret) + dev_err(udev->dev, "ctrlq send msg failed, ret = %d.\n", ret); + + return ret; +} + +static int udma_ctrlq_store_tpid_list(struct udma_dev *udev, + struct xarray *ctrlq_tpid_table, + struct udma_ctrlq_tpid_list_rsp *tpid_list_resp) +{ + int ret; + int i; + + if (debug_switch) + dev_info(udev->dev, "udma ctrlq store tpid list tp_list_cnt = %u.\n", + tpid_list_resp->tp_list_cnt); + + for (i = 0; i < (int)tpid_list_resp->tp_list_cnt; i++) { + ret = udma_ctrlq_store_one_tpid(udev, ctrlq_tpid_table, + &tpid_list_resp->tpid_list[i]); + if (ret) + goto err_store_one_tpid; + } + + return 0; + +err_store_one_tpid: + for (i--; i >= 0; i--) + udma_ctrlq_erase_one_tpid(ctrlq_tpid_table, tpid_list_resp->tpid_list[i].tpid); + + return ret; +} + +int udma_get_tp_list(struct ubcore_device *dev, struct ubcore_get_tp_cfg *tpid_cfg, + uint32_t *tp_cnt, struct ubcore_tp_info *tp_list, + struct ubcore_udata *udata) +{ + struct udma_ctrlq_get_tp_list_req_data tp_cfg_req = {}; + struct udma_ctrlq_tpid_list_rsp tpid_list_resp = {}; + struct udma_dev *udev = to_udma_dev(dev); + int ret; + int i; + + if (!udata) + tp_cfg_req.flag = UDMA_DEFAULT_PID; + else + tp_cfg_req.flag = (uint32_t)current->tgid & UDMA_PID_MASK; + + ret = udma_ctrlq_get_tpid_list(udev, &tp_cfg_req, tpid_cfg, &tpid_list_resp); + if (ret) { + dev_err(udev->dev, "udma ctrlq get tpid list failed, ret = %d.\n", ret); + return ret; + } + + if (tpid_list_resp.tp_list_cnt == 0 || tpid_list_resp.tp_list_cnt > *tp_cnt) { + dev_err(udev->dev, + "check tp list count failed, count = %u.\n", + tpid_list_resp.tp_list_cnt); + return -EINVAL; + } + + for (i = 0; i < tpid_list_resp.tp_list_cnt; i++) { + tp_list[i].tp_handle.bs.tpid = tpid_list_resp.tpid_list[i].tpid; + tp_list[i].tp_handle.bs.tpn_start = tpid_list_resp.tpid_list[i].tpn_start; + tp_list[i].tp_handle.bs.tp_cnt = + tpid_list_resp.tpid_list[i].tpn_cnt & UDMA_TPN_CNT_MASK; + } + *tp_cnt = tpid_list_resp.tp_list_cnt; + + ret = udma_ctrlq_store_tpid_list(udev, &udev->ctrlq_tpid_table, &tpid_list_resp); + if (ret) + dev_err(udev->dev, "udma ctrlq store list failed, ret = %d.\n", ret); + + return ret; +} + +void udma_ctrlq_destroy_tpid_list(struct udma_dev *dev, struct xarray *ctrlq_tpid_table, + bool is_need_flush) +{ + struct udma_ctrlq_tpid *tpid_entity = NULL; + unsigned long tpid = 0; + + xa_lock(ctrlq_tpid_table); + if (!xa_empty(ctrlq_tpid_table)) { + xa_for_each(ctrlq_tpid_table, tpid, tpid_entity) { + __xa_erase(ctrlq_tpid_table, tpid); + kfree(tpid_entity); + } + } + xa_unlock(ctrlq_tpid_table); + xa_destroy(ctrlq_tpid_table); +} + +static int udma_k_ctrlq_create_active_tp_msg(struct udma_dev *udev, + struct ubcore_active_tp_cfg *active_cfg, + uint32_t *tp_id) +{ + struct udma_ctrlq_active_tp_resp_data active_tp_resp = {}; + struct udma_ctrlq_active_tp_req_data active_tp_req = {}; + struct ubase_ctrlq_msg msg = {}; + int ret; + + active_tp_req.local_tp_id = active_cfg->tp_handle.bs.tpid; + active_tp_req.local_tpn_cnt = active_cfg->tp_handle.bs.tp_cnt; + active_tp_req.local_tpn_start = active_cfg->tp_handle.bs.tpn_start; + active_tp_req.local_psn = active_cfg->tp_attr.tx_psn; + + active_tp_req.remote_tp_id = active_cfg->peer_tp_handle.bs.tpid; + active_tp_req.remote_tpn_cnt = active_cfg->peer_tp_handle.bs.tp_cnt; + active_tp_req.remote_tpn_start = active_cfg->peer_tp_handle.bs.tpn_start; + active_tp_req.remote_psn = active_cfg->tp_attr.rx_psn; + + if (debug_switch) + udma_dfx_ctx_print(udev, "udma create active tp msg info", + active_tp_req.local_tp_id, + sizeof(struct udma_ctrlq_active_tp_req_data) / sizeof(uint32_t), + (uint32_t *)&active_tp_req); + + msg.opcode = UDMA_CMD_CTRLQ_ACTIVE_TP; + udma_ctrlq_set_tp_msg(&msg, (void *)&active_tp_req, sizeof(active_tp_req), + (void *)&active_tp_resp, sizeof(active_tp_resp)); + + ret = ubase_ctrlq_send_msg(udev->comdev.adev, &msg); + if (ret) + dev_err(udev->dev, "udma active tp send failed, ret = %d.\n", ret); + + *tp_id = active_tp_resp.local_tp_id; + + return ret; +} + +int udma_ctrlq_set_active_tp_ex(struct udma_dev *dev, + struct ubcore_active_tp_cfg *active_cfg) +{ + uint32_t tp_id = active_cfg->tp_handle.bs.tpid; + int ret; + + ret = udma_k_ctrlq_create_active_tp_msg(dev, active_cfg, &tp_id); + if (ret) + return ret; + + active_cfg->tp_handle.bs.tpid = tp_id; + + if (dev->is_ue) + (void)udma_send_req_to_mue(dev, &(active_cfg->tp_handle)); + + return 0; +} + +static int udma_k_ctrlq_deactive_tp(struct udma_dev *udev, union ubcore_tp_handle tp_handle, + struct ubcore_udata *udata) +{ +#define UDMA_RSP_TP_MUL 2 + uint32_t tp_id = tp_handle.bs.tpid & UDMA_TPHANDLE_TPID_SHIFT; + struct udma_ctrlq_deactive_tp_req_data deactive_tp_req = {}; + uint32_t tp_num = tp_handle.bs.tp_cnt; + struct ubase_ctrlq_msg msg = {}; + int ret; + + if (tp_num) { + ret = udma_close_ue_rx(udev, true, false, false, tp_num * UDMA_RSP_TP_MUL); + if (ret) { + dev_err(udev->dev, "close ue rx failed in deactivate tp.\n"); + return ret; + } + } + + msg.opcode = UDMA_CMD_CTRLQ_DEACTIVE_TP; + deactive_tp_req.tp_id = tp_id; + deactive_tp_req.tpn_cnt = tp_handle.bs.tp_cnt; + deactive_tp_req.start_tpn = tp_handle.bs.tpn_start; + if (!udata) + deactive_tp_req.pid_flag = UDMA_DEFAULT_PID; + else + deactive_tp_req.pid_flag = (uint32_t)current->tgid & UDMA_PID_MASK; + + udma_ctrlq_set_tp_msg(&msg, (void *)&deactive_tp_req, sizeof(deactive_tp_req), NULL, 0); + + ret = ubase_ctrlq_send_msg(udev->comdev.adev, &msg); + if (ret != -EAGAIN && ret) { + dev_err(udev->dev, "deactivate tp send msg failed, tp_id = %u, ret = %d.\n", + tp_id, ret); + if (tp_num) + udma_open_ue_rx(udev, true, false, false, tp_num * UDMA_RSP_TP_MUL); + return ret; + } + + udma_ctrlq_erase_one_tpid(&udev->ctrlq_tpid_table, tp_id); + + return (ret == -EAGAIN) ? 0 : ret; +} + +int udma_ctrlq_query_ubmem_info(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out) +{ +#define UDMA_CTRLQ_SER_TYPE_UBMEM 0x5 + struct udma_ctrlq_ubmem_out_query ubmem_info_out = {}; + struct udma_dev *udev = to_udma_dev(dev); + struct ubase_ctrlq_msg ctrlq_msg = {}; + uint32_t input_buf = 0; + int ret; + + if (out->addr == 0 || out->len != sizeof(struct udma_ctrlq_ubmem_out_query)) { + dev_err(udev->dev, "query ubmem info failed, addr is NULL:%d, len:%u.\n", + out->addr == 0, out->len); + return -EINVAL; + } + + ctrlq_msg.service_type = UDMA_CTRLQ_SER_TYPE_UBMEM; + ctrlq_msg.service_ver = UBASE_CTRLQ_SER_VER_01; + ctrlq_msg.need_resp = 1; + ctrlq_msg.in_size = sizeof(input_buf); + ctrlq_msg.in = (void *)&input_buf; + ctrlq_msg.out_size = sizeof(ubmem_info_out); + ctrlq_msg.out = &ubmem_info_out; + ctrlq_msg.opcode = UDMA_CTRLQ_QUERY_UBMEM_INFO; + + ret = ubase_ctrlq_send_msg(udev->comdev.adev, &ctrlq_msg); + if (ret) { + dev_err(udev->dev, "get dev res send ctrlq msg failed, ret is %d.\n", ret); + return ret; + } + + memcpy((void *)(uintptr_t)out->addr, &ubmem_info_out, sizeof(ubmem_info_out)); + + return ret; +} + +int udma_set_tp_attr(struct ubcore_device *dev, const uint64_t tp_handle, + const uint8_t tp_attr_cnt, const uint32_t tp_attr_bitmap, + const struct ubcore_tp_attr_value *tp_attr, struct ubcore_udata *udata) +{ + struct udma_ctrlq_set_tp_attr_req tp_attr_req = {}; + struct udma_dev *udev = to_udma_dev(dev); + union ubcore_tp_handle tp_handle_val; + struct ubase_ctrlq_msg msg = {}; + int ret; + + tp_handle_val.value = tp_handle; + tp_attr_req.tpid = tp_handle_val.bs.tpid; + tp_attr_req.tpn_cnt = tp_handle_val.bs.tp_cnt; + tp_attr_req.tpn_start = tp_handle_val.bs.tpn_start; + tp_attr_req.tp_attr_cnt = tp_attr_cnt; + tp_attr_req.tp_attr.tp_attr_bitmap = tp_attr_bitmap; + memcpy(&tp_attr_req.tp_attr.tp_attr_value, (void *)tp_attr, sizeof(*tp_attr)); + + udma_swap_endian((uint8_t *)tp_attr->sip, tp_attr_req.tp_attr.tp_attr_value.sip, + UBCORE_IP_ADDR_BYTES); + udma_swap_endian((uint8_t *)tp_attr->dip, tp_attr_req.tp_attr.tp_attr_value.dip, + UBCORE_IP_ADDR_BYTES); + udma_swap_endian((uint8_t *)tp_attr->sma, tp_attr_req.tp_attr.tp_attr_value.sma, + UBCORE_MAC_BYTES); + udma_swap_endian((uint8_t *)tp_attr->dma, tp_attr_req.tp_attr.tp_attr_value.dma, + UBCORE_MAC_BYTES); + udma_ctrlq_set_tp_msg(&msg, &tp_attr_req, sizeof(tp_attr_req), NULL, 0); + msg.opcode = UDMA_CMD_CTRLQ_SET_TP_ATTR; + + ret = ubase_ctrlq_send_msg(udev->comdev.adev, &msg); + if (ret) + dev_err(udev->dev, "set tp attr failed, tpid = %u, ret = %d.\n", + tp_attr_req.tpid, ret); + + return ret; +} + +int udma_get_tp_attr(struct ubcore_device *dev, const uint64_t tp_handle, + uint8_t *tp_attr_cnt, uint32_t *tp_attr_bitmap, + struct ubcore_tp_attr_value *tp_attr, struct ubcore_udata *udata) +{ + struct udma_ctrlq_get_tp_attr_resp tp_attr_resp = {}; + struct udma_ctrlq_get_tp_attr_req tp_attr_req = {}; + struct udma_dev *udev = to_udma_dev(dev); + union ubcore_tp_handle tp_handle_val; + struct ubase_ctrlq_msg msg = {}; + int ret; + + tp_handle_val.value = tp_handle; + tp_attr_req.tpid.tpid = tp_handle_val.bs.tpid; + tp_attr_req.tpid.tpn_cnt = tp_handle_val.bs.tp_cnt; + tp_attr_req.tpid.tpn_start = tp_handle_val.bs.tpn_start; + udma_ctrlq_set_tp_msg(&msg, &tp_attr_req, sizeof(tp_attr_req), &tp_attr_resp, + sizeof(tp_attr_resp)); + msg.opcode = UDMA_CMD_CTRLQ_GET_TP_ATTR; + + ret = ubase_ctrlq_send_msg(udev->comdev.adev, &msg); + if (ret) { + dev_err(udev->dev, "get tp attr failed, tpid = %u, ret = %d.\n", + tp_attr_req.tpid.tpid, ret); + return ret; + } + + *tp_attr_cnt = tp_attr_resp.tp_attr_cnt; + *tp_attr_bitmap = tp_attr_resp.tp_attr.tp_attr_bitmap; + memcpy((void *)tp_attr, &tp_attr_resp.tp_attr.tp_attr_value, + sizeof(tp_attr_resp.tp_attr.tp_attr_value)); + udma_swap_endian((uint8_t *)tp_attr_resp.tp_attr.tp_attr_value.sip, tp_attr->sip, + UBCORE_IP_ADDR_BYTES); + udma_swap_endian((uint8_t *)tp_attr_resp.tp_attr.tp_attr_value.dip, tp_attr->dip, + UBCORE_IP_ADDR_BYTES); + udma_swap_endian((uint8_t *)tp_attr_resp.tp_attr.tp_attr_value.sma, tp_attr->sma, + UBCORE_MAC_BYTES); + udma_swap_endian((uint8_t *)tp_attr_resp.tp_attr.tp_attr_value.dma, tp_attr->dma, + UBCORE_MAC_BYTES); + + return 0; +} + +int send_req_to_mue(struct udma_dev *udma_dev, struct ubcore_req *req, uint16_t opcode) +{ + struct udma_req_msg *req_msg; + struct ubase_cmd_buf in; + uint32_t msg_len; + int ret; + + msg_len = sizeof(*req_msg) + req->len; + req_msg = kzalloc(msg_len, GFP_KERNEL); + if (!req_msg) + return -ENOMEM; + + req_msg->resp_code = opcode; + + (void)memcpy(&req_msg->req, req, sizeof(*req)); + (void)memcpy(req_msg->req.data, req->data, req->len); + udma_fill_buf(&in, UBASE_OPC_UE_TO_MUE, false, msg_len, req_msg); + + ret = ubase_cmd_send_in(udma_dev->comdev.adev, &in); + if (ret) + dev_err(udma_dev->dev, + "send req msg cmd failed, ret is %d.\n", ret); + + kfree(req_msg); + + return ret; +} + +int send_resp_to_ue(struct udma_dev *udma_dev, struct ubcore_resp *req_host, + uint8_t dst_ue_idx, uint16_t opcode) +{ + struct udma_resp_msg *udma_req; + struct ubase_cmd_buf in; + uint32_t msg_len; + int ret; + + msg_len = sizeof(*udma_req) + req_host->len; + udma_req = kzalloc(msg_len, GFP_KERNEL); + if (!udma_req) + return -ENOMEM; + + udma_req->dst_ue_idx = dst_ue_idx; + udma_req->resp_code = opcode; + + (void)memcpy(&udma_req->resp, req_host, sizeof(*req_host)); + (void)memcpy(udma_req->resp.data, req_host->data, req_host->len); + + udma_fill_buf(&in, UBASE_OPC_MUE_TO_UE, false, msg_len, udma_req); + + ret = ubase_cmd_send_in(udma_dev->comdev.adev, &in); + if (ret) + dev_err(udma_dev->dev, + "send resp msg cmd failed, ret is %d.\n", ret); + + kfree(udma_req); + + return ret; +} + +int udma_active_tp(struct ubcore_device *dev, struct ubcore_active_tp_cfg *active_cfg) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + int ret; + + if (debug_switch) + udma_dfx_ctx_print(udma_dev, "udma active tp ex", active_cfg->tp_handle.bs.tpid, + sizeof(struct ubcore_active_tp_cfg) / sizeof(uint32_t), + (uint32_t *)active_cfg); + ret = udma_ctrlq_set_active_tp_ex(udma_dev, active_cfg); + if (ret) + dev_err(udma_dev->dev, "Failed to set active tp msg, ret %d.\n", ret); + + return ret; +} + +int udma_deactive_tp(struct ubcore_device *dev, union ubcore_tp_handle tp_handle, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + + if (debug_switch) + dev_info(udma_dev->dev, "udma deactivate tp ex tp_id = %u\n", tp_handle.bs.tpid); + + return udma_k_ctrlq_deactive_tp(udma_dev, tp_handle, udata); +} + +int udma_query_pair_dev_count(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out) +{ + struct udma_dev *udev = to_udma_dev(dev); + struct ubase_ctrlq_msg ctrlq_msg = {}; + struct ubase_bus_eid eid = {}; + uint32_t pair_device_num = 0; + int ret; + + if (out->addr == 0 || out->len != sizeof(pair_device_num)) { + dev_err(udev->dev, "query pair dev count, addr is NULL:%d, len:%u.\n", + out->addr == 0, out->len); + return -EINVAL; + } + + ret = ubase_get_bus_eid(udev->comdev.adev, &eid); + if (ret) { + dev_err(udev->dev, "get dev bus eid failed, ret is %d.\n", ret); + return ret; + } + + ctrlq_msg.service_type = UBASE_CTRLQ_SER_TYPE_DEV_REGISTER; + ctrlq_msg.service_ver = UBASE_CTRLQ_SER_VER_01; + ctrlq_msg.need_resp = 1; + ctrlq_msg.in_size = sizeof(eid); + ctrlq_msg.in = (void *)&eid; + ctrlq_msg.out_size = sizeof(pair_device_num); + ctrlq_msg.out = &pair_device_num; + ctrlq_msg.opcode = UDMA_CTRLQ_GET_DEV_RESOURCE_COUNT; + + ret = ubase_ctrlq_send_msg(udev->comdev.adev, &ctrlq_msg); + if (ret) { + dev_err(udev->dev, "get dev res send ctrlq msg failed, ret is %d.\n", ret); + return ret; + } + + memcpy((void *)(uintptr_t)out->addr, &pair_device_num, sizeof(pair_device_num)); + + return ret; +} diff --git a/drivers/ub/urma/hw/udma/udma_ctrlq_tp.h b/drivers/ub/urma/hw/udma/udma_ctrlq_tp.h new file mode 100644 index 0000000000000000000000000000000000000000..bdd4617cb4c448dc47468381de8401395b6143b8 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_ctrlq_tp.h @@ -0,0 +1,228 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_CTRLQ_TP_H__ +#define __UDMA_CTRLQ_TP_H__ + +#include "udma_common.h" + +#define UDMA_EID_SIZE 16 +#define UDMA_CNA_SIZE 16 +#define UDMA_PID_MASK 0xFFFFFF +#define UDMA_DEFAULT_PID 1 +#define UDMA_UE_NUM 64 +#define UDMA_MAX_UE_IDX 256 +#define UDMA_MAX_TPID_NUM 5 + +#define UDMA_CTRLQ_UBMEM_INFO_NUM (96) +#define UDMA_TPN_CNT_MASK 0x1F + +enum udma_ctrlq_cmd_code_type { + UDMA_CMD_CTRLQ_REMOVE_SINGLE_TP = 0x13, + UDMA_CMD_CTRLQ_TP_FLUSH_DONE, + UDMA_CMD_CTRLQ_CHECK_TP_ACTIVE, + UDMA_CMD_CTRLQ_GET_TP_LIST = 0x21, + UDMA_CMD_CTRLQ_ACTIVE_TP, + UDMA_CMD_CTRLQ_DEACTIVE_TP, + UDMA_CMD_CTRLQ_SET_TP_ATTR, + UDMA_CMD_CTRLQ_GET_TP_ATTR, + UDMA_CMD_CTRLQ_MAX +}; + +enum udma_ctrlq_ubmem_opcode { + UDMA_CTRLQ_QUERY_UBMEM_INFO = 0x1, +}; + +enum udma_ctrlq_trans_type { + UDMA_CTRLQ_TRANS_TYPE_TP_RM = 0, + UDMA_CTRLQ_TRANS_TYPE_CTP, + UDMA_CTRLQ_TRANS_TYPE_TP_UM, + UDMA_CTRLQ_TRANS_TYPE_TP_RC = 4, + UDMA_CTRLQ_TRANS_TYPE_MAX +}; + +enum udma_ctrlq_tpid_status { + UDMA_CTRLQ_TPID_IN_USE = 0, + UDMA_CTRLQ_TPID_EXITED, + UDMA_CTRLQ_TPID_IDLE, +}; + +struct udma_ctrlq_tpid { + uint32_t tpid : 24; + uint32_t tpn_cnt : 8; + uint32_t tpn_start : 24; + uint32_t rsv : 8; +}; + +struct udma_ctrlq_tpid_list_rsp { + uint32_t tp_list_cnt : 16; + uint32_t rsv : 16; + struct udma_ctrlq_tpid tpid_list[UDMA_MAX_TPID_NUM]; +}; + +struct udma_ctrlq_active_tp_req_data { + uint32_t local_tp_id : 24; + uint32_t local_tpn_cnt : 8; + uint32_t local_tpn_start : 24; + uint32_t rsv : 8; + uint32_t remote_tp_id : 24; + uint32_t remote_tpn_cnt : 8; + uint32_t remote_tpn_start : 24; + uint32_t rsv1 : 8; + uint32_t local_psn; + uint32_t remote_psn; +}; + +struct udma_ctrlq_active_tp_resp_data { + uint32_t local_tp_id : 24; + uint32_t local_tpn_cnt : 8; + uint32_t local_tpn_start : 24; + uint32_t rsv : 8; +}; + +struct udma_ctrlq_deactive_tp_req_data { + uint32_t tp_id : 24; + uint32_t tpn_cnt : 8; + uint32_t start_tpn : 24; + uint32_t rsv : 8; + uint32_t pid_flag : 24; + uint32_t rsv1 : 8; +}; + +struct udma_ctrlq_tp_flush_done_req_data { + uint32_t tpn : 24; + uint32_t rsv : 8; +}; + +struct udma_ctrlq_remove_single_tp_req_data { + uint32_t tpn : 24; + uint32_t tp_status : 8; +}; + +struct udma_ctrlq_tpn_data { + uint32_t tpg_flag : 8; + uint32_t rsv : 24; + uint32_t tpgn : 24; + uint32_t rsv1 : 8; + uint32_t tpn_cnt : 8; + uint32_t start_tpn : 24; +}; + +struct udma_ctrlq_check_tp_active_req_data { + uint32_t tp_id : 24; + uint32_t rsv : 8; + uint32_t pid_flag : 24; + uint32_t rsv1 : 8; +}; + +struct udma_ctrlq_check_tp_active_req_info { + uint32_t num : 8; + uint32_t rsv : 24; + struct udma_ctrlq_check_tp_active_req_data data[]; +}; + +struct udma_ctrlq_check_tp_active_rsp_data { + uint32_t tp_id : 24; + uint32_t result : 8; +}; + +struct udma_ctrlq_check_tp_active_rsp_info { + uint32_t num : 8; + uint32_t rsv : 24; + struct udma_ctrlq_check_tp_active_rsp_data data[]; +}; + +struct udma_ctrlq_get_tp_list_req_data { + uint8_t seid[UDMA_EID_SIZE]; + uint8_t deid[UDMA_EID_SIZE]; + uint32_t trans_type : 4; + uint32_t rsv : 4; + uint32_t flag : 24; +}; + +enum udma_cmd_ue_opcode { + UDMA_CMD_UBCORE_COMMAND = 0x1, + UDMA_CMD_NOTIFY_MUE_SAVE_TP = 0x2, + UDMA_CMD_NOTIFY_UE_FLUSH_DONE = 0x3, +}; + +struct udma_ue_tp_info { + uint32_t tp_cnt : 8; + uint32_t start_tpn : 24; +}; + +struct udma_ue_idx_table { + uint32_t num; + uint8_t ue_idx[UDMA_UE_NUM]; +}; + +struct udma_ctrlq_ubmem_out_query { + uint32_t data[UDMA_CTRLQ_UBMEM_INFO_NUM]; +}; + +struct udma_ctrlq_tp_attr { + uint32_t tp_attr_bitmap; + struct ubcore_tp_attr_value tp_attr_value; +}; + +struct udma_ctrlq_get_tp_attr_req { + struct udma_ctrlq_tpid tpid; +}; + +struct udma_ctrlq_set_tp_attr_req { + uint32_t tpid : 24; + uint32_t tpn_cnt : 8; + uint32_t tpn_start : 24; + uint32_t tp_attr_cnt : 8; + struct udma_ctrlq_tp_attr tp_attr; +}; + +struct udma_ctrlq_get_tp_attr_resp { + uint32_t tpid : 24; + uint32_t tp_attr_cnt : 8; + struct udma_ctrlq_tp_attr tp_attr; +}; + +struct udma_dev_resource_ratio { + struct ubase_bus_eid eid; + uint32_t index; +}; + +int udma_query_pair_dev_count(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out); + +int udma_get_dev_resource_ratio(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out); + +int udma_register_npu_cb(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out); + +int udma_unregister_npu_cb(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out); +int udma_ctrlq_tp_flush_done(struct udma_dev *udev, uint32_t tpn); +int udma_ctrlq_remove_single_tp(struct udma_dev *udev, uint32_t tpn, int status); +int udma_get_tp_list(struct ubcore_device *dev, struct ubcore_get_tp_cfg *tpid_cfg, + uint32_t *tp_cnt, struct ubcore_tp_info *tp_list, + struct ubcore_udata *udata); + +void udma_ctrlq_destroy_tpid_list(struct udma_dev *dev, struct xarray *ctrlq_tpid_table, + bool is_need_flush); +int udma_ctrlq_set_active_tp_ex(struct udma_dev *dev, + struct ubcore_active_tp_cfg *active_cfg); +int udma_ctrlq_query_ubmem_info(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out); + +int udma_set_tp_attr(struct ubcore_device *dev, const uint64_t tp_handle, + const uint8_t tp_attr_cnt, const uint32_t tp_attr_bitmap, + const struct ubcore_tp_attr_value *tp_attr, struct ubcore_udata *udata); +int udma_get_tp_attr(struct ubcore_device *dev, const uint64_t tp_handle, + uint8_t *tp_attr_cnt, uint32_t *tp_attr_bitmap, + struct ubcore_tp_attr_value *tp_attr, struct ubcore_udata *udata); +int send_resp_to_ue(struct udma_dev *udma_dev, struct ubcore_resp *req_host, + uint8_t dst_ue_idx, uint16_t opcode); +int send_req_to_mue(struct udma_dev *udma_dev, struct ubcore_req *req, uint16_t opcode); +int udma_active_tp(struct ubcore_device *dev, struct ubcore_active_tp_cfg *active_cfg); +int udma_deactive_tp(struct ubcore_device *dev, union ubcore_tp_handle tp_handle, + struct ubcore_udata *udata); + +#endif /* __UDMA_CTRLQ_TP_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_ctx.c b/drivers/ub/urma/hw/udma/udma_ctx.c new file mode 100644 index 0000000000000000000000000000000000000000..ccc3b4905af9c9bd42f9b3b8660939bcf69ce3fc --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_ctx.c @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt + +#include +#include +#include "udma_jfs.h" +#include "udma_jetty.h" +#include "udma_ctrlq_tp.h" +#include "udma_ctx.h" + +static int udma_init_ctx_resp(struct udma_dev *dev, struct ubcore_udrv_priv *udrv_data) +{ + struct udma_create_ctx_resp resp; + unsigned long byte; + + if (!udrv_data->out_addr || + udrv_data->out_len < sizeof(resp)) { + dev_err(dev->dev, + "Invalid ctx resp out: len %d or addr is invalid.\n", + udrv_data->out_len); + return -EINVAL; + } + + resp.cqe_size = dev->caps.cqe_size; + resp.dwqe_enable = !!(dev->caps.feature & UDMA_CAP_FEATURE_DIRECT_WQE); + resp.reduce_enable = !!(dev->caps.feature & UDMA_CAP_FEATURE_REDUCE); + resp.ue_id = dev->ue_id; + resp.chip_id = dev->chip_id; + resp.die_id = dev->die_id; + resp.dump_aux_info = dump_aux_info; + resp.jfr_sge = dev->caps.jfr_sge; + resp.hugepage_enable = ubase_adev_prealloc_supported(dev->comdev.adev); + + byte = copy_to_user((void *)(uintptr_t)udrv_data->out_addr, &resp, + (uint32_t)sizeof(resp)); + if (byte) { + dev_err(dev->dev, + "copy ctx resp to user failed, byte = %lu.\n", byte); + return -EFAULT; + } + + return 0; +} + +struct ubcore_ucontext *udma_alloc_ucontext(struct ubcore_device *ub_dev, + uint32_t eid_index, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *dev = to_udma_dev(ub_dev); + struct udma_context *ctx; + int ret; + + ctx = kzalloc(sizeof(struct udma_context), GFP_KERNEL); + if (ctx == NULL) + return NULL; + + ctx->sva = ummu_sva_bind_device(dev->dev, current->mm, NULL); + if (!ctx->sva) { + dev_err(dev->dev, "SVA failed to bind device.\n"); + goto err_free_ctx; + } + + ret = ummu_get_tid(dev->dev, ctx->sva, &ctx->tid); + if (ret) { + dev_err(dev->dev, "Failed to get tid.\n"); + goto err_unbind_dev; + } + + ctx->dev = dev; + INIT_LIST_HEAD(&ctx->pgdir_list); + mutex_init(&ctx->pgdir_mutex); + INIT_LIST_HEAD(&ctx->hugepage_list); + mutex_init(&ctx->hugepage_lock); + + ret = udma_init_ctx_resp(dev, udrv_data); + if (ret) { + dev_err(dev->dev, "Init ctx resp failed.\n"); + goto err_init_ctx_resp; + } + + return &ctx->base; + +err_init_ctx_resp: + mutex_destroy(&ctx->hugepage_lock); + mutex_destroy(&ctx->pgdir_mutex); +err_unbind_dev: + ummu_sva_unbind_device(ctx->sva); +err_free_ctx: + kfree(ctx); + return NULL; +} + +int udma_free_ucontext(struct ubcore_ucontext *ucontext) +{ + struct udma_dev *udma_dev = to_udma_dev(ucontext->ub_dev); + struct udma_hugepage_priv *priv; + struct vm_area_struct *vma; + struct udma_context *ctx; + int ret; + int i; + + ctx = to_udma_context(ucontext); + + ret = ummu_core_invalidate_cfg_table(ctx->tid); + if (ret) + dev_err(udma_dev->dev, "invalidate cfg_table failed, ret=%d.\n", ret); + + mutex_destroy(&ctx->pgdir_mutex); + ummu_sva_unbind_device(ctx->sva); + + mutex_lock(&ctx->hugepage_lock); + list_for_each_entry(priv, &ctx->hugepage_list, list) { + if (current->mm) { + mmap_write_lock(current->mm); + vma = find_vma(current->mm, (unsigned long)priv->va_base); + if (vma != NULL && vma->vm_start <= (unsigned long)priv->va_base && + vma->vm_end >= (unsigned long)(priv->va_base + priv->va_len)) + zap_vma_ptes(vma, (unsigned long)priv->va_base, priv->va_len); + mmap_write_unlock(current->mm); + } + + dev_info(udma_dev->dev, "unmap_hugepage, 2m_page_num=%u.\n", priv->page_num); + for (i = 0; i < priv->page_num; i++) + __free_pages(priv->pages[i], get_order(UDMA_HUGEPAGE_SIZE)); + kfree(priv->pages); + kfree(priv); + } + mutex_unlock(&ctx->hugepage_lock); + mutex_destroy(&ctx->hugepage_lock); + + kfree(ctx); + + return 0; +} + +static int udma_mmap_jetty_dsqe(struct udma_dev *dev, struct ubcore_ucontext *uctx, + struct vm_area_struct *vma) +{ + struct ubcore_ucontext *jetty_uctx; + struct udma_jetty_queue *sq; + uint64_t address; + uint64_t j_id; + + j_id = get_mmap_idx(vma); + + xa_lock(&dev->jetty_table.xa); + sq = xa_load(&dev->jetty_table.xa, j_id); + if (!sq) { + dev_err(dev->dev, + "mmap failed, j_id: %llu not exist\n", j_id); + xa_unlock(&dev->jetty_table.xa); + return -EINVAL; + } + + if (sq->is_jetty) + jetty_uctx = to_udma_jetty_from_queue(sq)->ubcore_jetty.uctx; + else + jetty_uctx = to_udma_jfs_from_queue(sq)->ubcore_jfs.uctx; + + if (jetty_uctx != uctx) { + dev_err(dev->dev, + "mmap failed, j_id: %llu, uctx invalid\n", j_id); + xa_unlock(&dev->jetty_table.xa); + return -EINVAL; + } + xa_unlock(&dev->jetty_table.xa); + + address = (uint64_t)dev->db_base + JETTY_DSQE_OFFSET + j_id * UDMA_HW_PAGE_SIZE; + + if (io_remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT, + PAGE_SIZE, vma->vm_page_prot)) + return -EAGAIN; + + return 0; +} + +static int udma_mmap_hugepage(struct udma_dev *dev, struct ubcore_ucontext *uctx, + struct vm_area_struct *vma) +{ + uint32_t max_map_size = dev->caps.cqe_size * dev->caps.jfc.depth; + uint32_t map_size = vma->vm_end - vma->vm_start; + + if (!IS_ALIGNED(map_size, UDMA_HUGEPAGE_SIZE)) { + dev_err(dev->dev, "mmap size is not 2m alignment.\n"); + return -EINVAL; + } + + if (map_size == 0) { + dev_err(dev->dev, "mmap size is zero.\n"); + return -EINVAL; + } + + if (map_size > max_map_size) { + dev_err(dev->dev, "mmap size(%u) is greater than the max_size.\n", + map_size); + return -EINVAL; + } + + vm_flags_set(vma, VM_IO | VM_LOCKED | VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY); + vma->vm_page_prot = __pgprot(((~PTE_ATTRINDX_MASK) & vma->vm_page_prot.pgprot) | + PTE_ATTRINDX(MT_NORMAL)); + if (udma_alloc_u_hugepage(to_udma_context(uctx), vma)) { + dev_err(dev->dev, "failed to alloc hugepage.\n"); + return -ENOMEM; + } + + return 0; +} + +int udma_mmap(struct ubcore_ucontext *uctx, struct vm_area_struct *vma) +{ +#define JFC_DB_UNMAP_BOUND 1 + struct udma_dev *udma_dev = to_udma_dev(uctx->ub_dev); + uint32_t cmd; + + if (((vma->vm_end - vma->vm_start) % PAGE_SIZE) != 0) { + dev_err(udma_dev->dev, + "mmap failed, unexpected vm area size.\n"); + return -EINVAL; + } + + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + cmd = get_mmap_cmd(vma); + switch (cmd) { + case UDMA_MMAP_JFC_PAGE: + if (io_remap_pfn_range(vma, vma->vm_start, + jfc_arm_mode > JFC_DB_UNMAP_BOUND ? + (uint64_t)udma_dev->db_base >> PAGE_SHIFT : + page_to_pfn(udma_dev->db_page), + PAGE_SIZE, vma->vm_page_prot)) + return -EAGAIN; + break; + case UDMA_MMAP_JETTY_DSQE: + return udma_mmap_jetty_dsqe(udma_dev, uctx, vma); + case UDMA_MMAP_HUGEPAGE: + return udma_mmap_hugepage(udma_dev, uctx, vma); + default: + dev_err(udma_dev->dev, + "mmap failed, cmd(%u) not support\n", cmd); + return -EINVAL; + } + + return 0; +} + +int udma_alloc_u_hugepage(struct udma_context *ctx, struct vm_area_struct *vma) +{ + uint32_t page_num = (vma->vm_end - vma->vm_start) >> UDMA_HUGEPAGE_SHIFT; + struct udma_hugepage_priv *priv; + int ret = -ENOMEM; + int i; + + mutex_lock(&ctx->dev->hugepage_lock); + if (page_num > ctx->dev->total_hugepage_num) { + dev_err(ctx->dev->dev, "insufficient resources for mmap.\n"); + mutex_unlock(&ctx->dev->hugepage_lock); + return -EINVAL; + } + ctx->dev->total_hugepage_num -= page_num; + mutex_unlock(&ctx->dev->hugepage_lock); + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + goto err_alloc_priv; + + priv->page_num = page_num; + priv->pages = kcalloc(priv->page_num, sizeof(*priv->pages), GFP_KERNEL); + if (!priv->pages) + goto err_alloc_arr; + + for (i = 0; i < priv->page_num; i++) { + priv->pages[i] = alloc_pages(GFP_KERNEL | __GFP_ZERO, + get_order(UDMA_HUGEPAGE_SIZE)); + if (!priv->pages[i]) { + dev_err(ctx->dev->dev, "failed to alloc 2M pages.\n"); + goto err_alloc_pages; + } + ret = remap_pfn_range(vma, vma->vm_start + i * UDMA_HUGEPAGE_SIZE, + page_to_pfn(priv->pages[i]), UDMA_HUGEPAGE_SIZE, + vma->vm_page_prot); + if (ret) { + dev_err(ctx->dev->dev, "failed to remap_pfn_range, ret=%d.\n", ret); + goto err_remap_pfn_range; + } + } + + priv->va_base = (void *)vma->vm_start; + priv->va_len = priv->page_num << UDMA_HUGEPAGE_SHIFT; + priv->left_va_len = priv->va_len; + refcount_set(&priv->refcnt, 1); + + mutex_lock(&ctx->hugepage_lock); + list_add(&priv->list, &ctx->hugepage_list); + mutex_unlock(&ctx->hugepage_lock); + + if (dfx_switch) + dev_info_ratelimited(ctx->dev->dev, "map_hugepage, 2m_page_num=%u.\n", + priv->page_num); + return 0; + +err_remap_pfn_range: +err_alloc_pages: + for (i = 0; i < priv->page_num; i++) { + if (priv->pages[i]) + __free_pages(priv->pages[i], get_order(UDMA_HUGEPAGE_SIZE)); + else + break; + } + kfree(priv->pages); +err_alloc_arr: + kfree(priv); +err_alloc_priv: + mutex_lock(&ctx->dev->hugepage_lock); + ctx->dev->total_hugepage_num += page_num; + mutex_unlock(&ctx->dev->hugepage_lock); + + return ret; +} + +static struct udma_hugepage_priv *udma_list_find_before(struct udma_context *ctx, void *va) +{ + struct udma_hugepage_priv *priv; + + list_for_each_entry(priv, &ctx->hugepage_list, list) { + if (va >= priv->va_base && va < priv->va_base + priv->va_len) + return priv; + } + + return NULL; +} + +int udma_occupy_u_hugepage(struct udma_context *ctx, void *va) +{ + struct udma_hugepage_priv *priv; + + mutex_lock(&ctx->hugepage_lock); + priv = udma_list_find_before(ctx, va); + if (priv) { + if (dfx_switch) + dev_info_ratelimited(ctx->dev->dev, "occupy_hugepage.\n"); + refcount_inc(&priv->refcnt); + } + mutex_unlock(&ctx->hugepage_lock); + + return priv ? 0 : -EFAULT; +} + +void udma_return_u_hugepage(struct udma_context *ctx, void *va) +{ + struct udma_hugepage_priv *priv; + struct vm_area_struct *vma; + uint32_t i; + + mutex_lock(&ctx->hugepage_lock); + priv = udma_list_find_before(ctx, va); + if (!priv) { + mutex_unlock(&ctx->hugepage_lock); + dev_warn(ctx->dev->dev, "va is invalid addr.\n"); + return; + } + + if (dfx_switch) + dev_info_ratelimited(ctx->dev->dev, "return_hugepage.\n"); + refcount_dec(&priv->refcnt); + if (!refcount_dec_if_one(&priv->refcnt)) { + mutex_unlock(&ctx->hugepage_lock); + return; + } + + list_del(&priv->list); + mutex_unlock(&ctx->hugepage_lock); + + if (current->mm) { + mmap_write_lock(current->mm); + vma = find_vma(current->mm, (unsigned long)priv->va_base); + if (vma != NULL && vma->vm_start <= (unsigned long)priv->va_base && + vma->vm_end >= (unsigned long)(priv->va_base + priv->va_len)) + zap_vma_ptes(vma, (unsigned long)priv->va_base, priv->va_len); + mmap_write_unlock(current->mm); + } else { + dev_warn_ratelimited(ctx->dev->dev, "current mm released.\n"); + } + + if (dfx_switch) + dev_info_ratelimited(ctx->dev->dev, "unmap_hugepage, 2m_page_num=%u.\n", + priv->page_num); + mutex_lock(&ctx->dev->hugepage_lock); + for (i = 0; i < priv->page_num; i++) + __free_pages(priv->pages[i], get_order(UDMA_HUGEPAGE_SIZE)); + ctx->dev->total_hugepage_num += priv->page_num; + mutex_unlock(&ctx->dev->hugepage_lock); + kfree(priv->pages); + kfree(priv); +} diff --git a/drivers/ub/urma/hw/udma/udma_ctx.h b/drivers/ub/urma/hw/udma/udma_ctx.h new file mode 100644 index 0000000000000000000000000000000000000000..2521d2de310850b7bbd5b2ce26e3f7cbe5787488 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_ctx.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_CTX_H__ +#define __UDMA_CTX_H__ + +#include +#include +#include "udma_dev.h" + +struct udma_context { + struct ubcore_ucontext base; + struct udma_dev *dev; + uint32_t uasid; + struct list_head pgdir_list; + struct mutex pgdir_mutex; + struct iommu_sva *sva; + uint32_t tid; + struct mutex hugepage_lock; + struct list_head hugepage_list; +}; + +static inline struct udma_context *to_udma_context(struct ubcore_ucontext *uctx) +{ + return container_of(uctx, struct udma_context, base); +} + +static inline uint64_t get_mmap_idx(struct vm_area_struct *vma) +{ + return (vma->vm_pgoff >> MAP_INDEX_SHIFT) & MAP_INDEX_MASK; +} + +static inline int get_mmap_cmd(struct vm_area_struct *vma) +{ + return (vma->vm_pgoff & MAP_COMMAND_MASK); +} + +struct ubcore_ucontext *udma_alloc_ucontext(struct ubcore_device *ub_dev, + uint32_t eid_index, + struct ubcore_udrv_priv *udrv_data); +int udma_free_ucontext(struct ubcore_ucontext *ucontext); +int udma_mmap(struct ubcore_ucontext *uctx, struct vm_area_struct *vma); + +int udma_alloc_u_hugepage(struct udma_context *ctx, struct vm_area_struct *vma); +int udma_occupy_u_hugepage(struct udma_context *ctx, void *va); +void udma_return_u_hugepage(struct udma_context *ctx, void *va); + +#endif /* __UDMA_CTX_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_db.c b/drivers/ub/urma/hw/udma/udma_db.c new file mode 100644 index 0000000000000000000000000000000000000000..c66d6b23b2e8a2aa41f805637cbb67381efc930d --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_db.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt + +#include +#include +#include "udma_common.h" +#include "udma_db.h" + +int udma_pin_sw_db(struct udma_context *ctx, struct udma_sw_db *db) +{ + uint64_t page_addr = db->db_addr & PAGE_MASK; + struct udma_sw_db_page *page; + struct udma_umem_param param; + uint32_t offset = 0; + int ret = 0; + + param.ub_dev = &ctx->dev->ub_dev; + param.va = page_addr; + param.len = PAGE_SIZE; + param.flag.bs.writable = 1; + param.flag.bs.non_pin = 0; + param.is_kernel = false; + offset = db->db_addr - page_addr; + + mutex_lock(&ctx->pgdir_mutex); + + list_for_each_entry(page, &ctx->pgdir_list, list) { + if (page->user_virt == page_addr) + goto found; + } + + page = kmalloc(sizeof(*page), GFP_KERNEL); + if (!page) { + ret = -ENOMEM; + goto out; + } + + refcount_set(&page->refcount, 1); + page->user_virt = page_addr; + page->umem = udma_umem_get(¶m); + if (IS_ERR(page->umem)) { + ret = PTR_ERR(page->umem); + dev_err(ctx->dev->dev, "Failed to get umem, ret: %d.\n", ret); + kfree(page); + goto out; + } + + list_add(&page->list, &ctx->pgdir_list); + db->page = page; + db->virt_addr = (char *)sg_virt(page->umem->sg_head.sgl) + offset; + mutex_unlock(&ctx->pgdir_mutex); + return 0; +found: + db->page = page; + db->virt_addr = (char *)sg_virt(page->umem->sg_head.sgl) + offset; + refcount_inc(&page->refcount); +out: + mutex_unlock(&ctx->pgdir_mutex); + return ret; +} + +void udma_unpin_sw_db(struct udma_context *ctx, struct udma_sw_db *db) +{ + mutex_lock(&ctx->pgdir_mutex); + + if (refcount_dec_and_test(&db->page->refcount)) { + list_del(&db->page->list); + udma_umem_release(db->page->umem, false); + kfree(db->page); + } + + mutex_unlock(&ctx->pgdir_mutex); +} + +static int udma_alloc_db_from_page(struct udma_k_sw_db_page *page, + struct udma_sw_db *db, enum udma_db_type type) +{ + uint32_t index; + + index = find_first_bit(page->bitmap, page->num_db); + if (index >= page->num_db) + return -ENOMEM; + + clear_bit(index, page->bitmap); + + db->index = index; + db->kpage = page; + db->type = type; + db->db_addr = page->db_buf.addr + db->index * UDMA_DB_SIZE; + db->db_record = page->db_buf.kva + db->index * UDMA_DB_SIZE; + *db->db_record = 0; + + return 0; +} + +static struct udma_k_sw_db_page *udma_alloc_db_page(struct udma_dev *dev, + enum udma_db_type type) +{ + struct udma_k_sw_db_page *page; + int ret; + + page = kzalloc(sizeof(*page), GFP_KERNEL); + if (!page) + return NULL; + + page->num_db = PAGE_SIZE / UDMA_DB_SIZE; + + page->bitmap = bitmap_alloc(page->num_db, GFP_KERNEL); + if (!page->bitmap) { + dev_err(dev->dev, "Failed alloc db bitmap, db type is %u.\n", type); + goto err_bitmap; + } + + bitmap_fill(page->bitmap, page->num_db); + + ret = udma_alloc_normal_buf(dev, PAGE_SIZE, &page->db_buf); + if (ret) { + dev_err(dev->dev, "Failed alloc db page buf, ret is %d.\n", ret); + goto err_kva; + } + + return page; +err_kva: + bitmap_free(page->bitmap); +err_bitmap: + kfree(page); + return NULL; +} + +int udma_alloc_sw_db(struct udma_dev *dev, struct udma_sw_db *db, + enum udma_db_type type) +{ + struct udma_k_sw_db_page *page; + int ret = 0; + + mutex_lock(&dev->db_mutex); + + list_for_each_entry(page, &dev->db_list[type], list) + if (!udma_alloc_db_from_page(page, db, type)) + goto out; + + page = udma_alloc_db_page(dev, type); + if (!page) { + ret = -ENOMEM; + dev_err(dev->dev, "Failed alloc sw db page db_type = %u\n", type); + goto out; + } + + list_add(&page->list, &dev->db_list[type]); + + /* This should never fail */ + (void)udma_alloc_db_from_page(page, db, type); +out: + mutex_unlock(&dev->db_mutex); + + return ret; +} + +void udma_free_sw_db(struct udma_dev *dev, struct udma_sw_db *db) +{ + mutex_lock(&dev->db_mutex); + + set_bit(db->index, db->kpage->bitmap); + + if (bitmap_full(db->kpage->bitmap, db->kpage->num_db)) { + udma_free_normal_buf(dev, PAGE_SIZE, &db->kpage->db_buf); + bitmap_free(db->kpage->bitmap); + list_del(&db->kpage->list); + kfree(db->kpage); + db->kpage = NULL; + } + + mutex_unlock(&dev->db_mutex); +} diff --git a/drivers/ub/urma/hw/udma/udma_db.h b/drivers/ub/urma/hw/udma/udma_db.h new file mode 100644 index 0000000000000000000000000000000000000000..0fe018304149bf9092c4fdfd8ec13f77f5419a9c --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_db.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_DB_H__ +#define __UDMA_DB_H__ + +#include "udma_ctx.h" +#include "udma_dev.h" + +int udma_pin_sw_db(struct udma_context *ctx, struct udma_sw_db *db); +void udma_unpin_sw_db(struct udma_context *ctx, struct udma_sw_db *db); +int udma_alloc_sw_db(struct udma_dev *dev, struct udma_sw_db *db, + enum udma_db_type type); +void udma_free_sw_db(struct udma_dev *dev, struct udma_sw_db *db); + +#endif /* __UDMA_DB_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_debugfs.c b/drivers/ub/urma/hw/udma/udma_debugfs.c new file mode 100644 index 0000000000000000000000000000000000000000..efc94a3d7e9efd1d44c3a9398462a56493e33c0d --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_debugfs.c @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt +#define pr_fmt(fmt) "UDMA: " fmt + +#include +#include +#include +#include +#include "udma_cmd.h" +#include "udma_debugfs.h" + +static struct dentry *g_udma_dbgfs_root; + +static struct udma_debugfs_file_info g_ta_dfx_mod[TA_MAX_SIZE] = { + {"mrd", RDONLY, UDMA_CMD_DEBUGFS_TA_INFO, UDMA_TA_MRD}, +}; + +static struct udma_debugfs_file_info g_tp_dfx_mod[TP_MAX_SIZE] = { + {"rxtx", RDONLY, UDMA_CMD_DEBUGFS_TP_INFO, UDMA_TP_RXTX}, +}; + +static void show_ta_mrd_dfx(struct udma_query_mrd_dfx *data) +{ + pr_info("****************** ta_mrd_dfx ******************\n"); + pr_info("mrd_dsqe_issue_cnt\t0x%08x\n", data->mrd_dsqe_issue_cnt); + pr_info("mrd_dsqe_exec_cnt\t0x%08x\n", data->mrd_dsqe_exec_cnt); + pr_info("mrd_dsqe_drop_cnt\t0x%08x\n", data->mrd_dsqe_drop_cnt); + pr_info("mrd_jfsdb_issue_cnt\t0x%08x\n", data->mrd_jfsdb_issue_cnt); + pr_info("mrd_jfsdb_exec_cnt\t0x%08x\n", data->mrd_jfsdb_exec_cnt); + pr_info("mrd_mb_issue_cnt\t\t0x%08x\n", data->mrd_mb_issue_cnt); + pr_info("mrd_mb_exec_cnt\t\t0x%08x\n", data->mrd_mb_exec_cnt); + pr_info("mrd_eqdb_issue_cnt\t0x%08x\n", data->mrd_eqdb_issue_cnt); + pr_info("mrd_mb_buff_full\t\t0x%08x\n", data->mrd_mb_buff_full); + pr_info("mrd_mb_buff_empty\t0x%08x\n", data->mrd_mb_buff_empty); + pr_info("mrd_mem_ecc_err_1b\t0x%08x\n", data->mrd_mem_ecc_err_1b); + pr_info("mrd_mem_ecc_1b_info\t0x%08x\n", data->mrd_mem_ecc_1b_info); + pr_info("mrd_mb_state\t\t0x%08x\n", data->mrd_mb_state); + pr_info("mrd_eqdb_exec_cnt\t0x%08x\n", data->mrd_eqdb_exec_cnt); + pr_info("****************** ta_mrd_dfx ******************\n"); +} + +static void show_tp_rxtx_dfx(struct udma_query_rxtx_dfx *data) +{ + pr_info("****************** tp_rxtx_dfx ******************\n"); + pr_info("tpp2_txdma_hdr_um_pkt_cnt\t0x%016llx\n", data->tpp2_txdma_hdr_um_pkt_cnt); + pr_info("tpp2_txdma_ctp_rm_pkt_cnt\t0x%016llx\n", data->tpp2_txdma_ctp_rm_pkt_cnt); + pr_info("tpp2_txdma_ctp_rc_pkt_cnt\t0x%016llx\n", data->tpp2_txdma_ctp_rc_pkt_cnt); + pr_info("tpp2_txdma_tp_rm_pkt_cnt\t\t0x%016llx\n", data->tpp2_txdma_tp_rm_pkt_cnt); + pr_info("tpp2_txdma_tp_rc_pkt_cnt\t\t0x%016llx\n", data->tpp2_txdma_tp_rc_pkt_cnt); + pr_info("rhp_glb_rm_pkt_cnt\t\t0x%016llx\n", data->rhp_glb_rm_pkt_cnt); + pr_info("rhp_glb_rc_pkt_cnt\t\t0x%016llx\n", data->rhp_glb_rc_pkt_cnt); + pr_info("rhp_clan_rm_pkt_cnt\t\t0x%016llx\n", data->rhp_clan_rm_pkt_cnt); + pr_info("rhp_clan_rc_pkt_cnt\t\t0x%016llx\n", data->rhp_clan_rc_pkt_cnt); + pr_info("rhp_ud_pkt_cnt\t\t\t0x%016llx\n", data->rhp_ud_pkt_cnt); + pr_info("****************** tp_rxtx_dfx ******************\n"); +} + +static int udma_query_mrd_dfx(struct file_private_data *private_data) +{ + struct udma_query_mrd_dfx out_regs; + struct ubase_cmd_buf in, out; + int ret; + + out_regs.sub_module = private_data->sub_opcode; + udma_fill_buf(&in, private_data->opcode, true, + sizeof(struct udma_query_mrd_dfx), &out_regs); + udma_fill_buf(&out, private_data->opcode, true, + sizeof(struct udma_query_mrd_dfx), &out_regs); + + ret = ubase_cmd_send_inout(private_data->udma_dev->comdev.adev, &in, &out); + if (ret) { + dev_err(private_data->udma_dev->dev, "failed to query mrd DFX, ret = %d.\n", ret); + return ret; + } + + show_ta_mrd_dfx(&out_regs); + + return 0; +} + +static int udma_query_rxtx_dfx(struct file_private_data *private_data) +{ + struct udma_query_rxtx_dfx out_regs; + struct ubase_cmd_buf in, out; + int ret; + + out_regs.sub_module = private_data->sub_opcode; + udma_fill_buf(&in, private_data->opcode, true, sizeof(struct udma_query_rxtx_dfx), + &out_regs); + udma_fill_buf(&out, private_data->opcode, true, + sizeof(struct udma_query_rxtx_dfx), &out_regs); + + ret = ubase_cmd_send_inout(private_data->udma_dev->comdev.adev, &in, &out); + if (ret) { + dev_err(private_data->udma_dev->dev, "failed to query rxtx DFX, ret = %d.\n", ret); + return ret; + } + + show_tp_rxtx_dfx(&out_regs); + + return 0; +} + +static inline int udma_debugfs_open(struct inode *inode, struct file *filp) +{ + filp->private_data = inode->i_private; + + return 0; +} + +static ssize_t udma_debugfs_read(struct file *filp, char __user *buf, + size_t size, loff_t *ppos) +{ + struct file_private_data *private_data = filp->private_data; + int ret; + + switch (private_data->sub_opcode) { + case UDMA_TA_MRD: + ret = udma_query_mrd_dfx(private_data); + break; + case UDMA_TP_RXTX: + ret = udma_query_rxtx_dfx(private_data); + break; + default: + dev_err(private_data->udma_dev->dev, "invalid type %u.\n", + private_data->sub_opcode); + return -EFAULT; + } + + return ret; +} + +static const struct file_operations udma_debugfs_rd_fops = { + .owner = THIS_MODULE, + .read = udma_debugfs_read, + .open = udma_debugfs_open, +}; + +static const uint16_t file_mod[FILE_MOD_SIZE] = { + 0200, 0400, +}; + +static int udma_debugfs_create_files(struct udma_dev *udma_dev, struct udma_dev_debugfs *dbgfs) +{ + struct file_private_data *private_data; + struct file_private_data *cur_p; + struct dentry *entry; + int i; + + private_data = kzalloc(sizeof(struct file_private_data) * (TA_MAX_SIZE + TP_MAX_SIZE), + GFP_KERNEL); + if (!private_data) + return -ENOMEM; + + for (i = 0; i < TA_MAX_SIZE; ++i) { + cur_p = private_data + i; + cur_p->udma_dev = udma_dev; + cur_p->opcode = g_ta_dfx_mod[i].opcode; + cur_p->sub_opcode = g_ta_dfx_mod[i].sub_opcode; + entry = debugfs_create_file(g_ta_dfx_mod[i].name, file_mod[g_ta_dfx_mod[i].fmod], + dbgfs->ta_root, cur_p, &udma_debugfs_rd_fops); + if (IS_ERR(entry)) { + dev_err(udma_dev->dev, "create %s failed.\n", g_ta_dfx_mod[i].name); + kfree(private_data); + return -EINVAL; + } + } + + for (i = 0; i < TP_MAX_SIZE; ++i) { + cur_p = private_data + i + TA_MAX_SIZE; + cur_p->udma_dev = udma_dev; + cur_p->opcode = g_tp_dfx_mod[i].opcode; + cur_p->sub_opcode = g_tp_dfx_mod[i].sub_opcode; + entry = debugfs_create_file(g_tp_dfx_mod[i].name, file_mod[g_tp_dfx_mod[i].fmod], + dbgfs->tp_root, cur_p, &udma_debugfs_rd_fops); + if (IS_ERR(entry)) { + dev_err(udma_dev->dev, "create %s failed.\n", g_tp_dfx_mod[i].name); + kfree(private_data); + return -EINVAL; + } + } + + dbgfs->private_data = private_data; + dbgfs->private_data_size = TA_MAX_SIZE + TP_MAX_SIZE; + + return 0; +} + +void udma_register_debugfs(struct udma_dev *udma_dev) +{ + struct udma_dev_debugfs *dbgfs; + + if (IS_ERR_OR_NULL(g_udma_dbgfs_root)) { + dev_err(udma_dev->dev, "Debugfs root path does not exist.\n"); + goto create_error; + } + + dbgfs = kzalloc(sizeof(*dbgfs), GFP_KERNEL); + if (!dbgfs) + goto create_error; + + dbgfs->root = debugfs_create_dir(udma_dev->dev_name, g_udma_dbgfs_root); + if (IS_ERR(dbgfs->root)) { + dev_err(udma_dev->dev, "Debugfs create dev path failed.\n"); + goto create_dev_error; + } + + dbgfs->ta_root = debugfs_create_dir("ta", dbgfs->root); + if (IS_ERR(dbgfs->ta_root)) { + dev_err(udma_dev->dev, "Debugfs create ta path failed.\n"); + goto create_path_error; + } + + dbgfs->tp_root = debugfs_create_dir("tp", dbgfs->root); + if (IS_ERR(dbgfs->tp_root)) { + dev_err(udma_dev->dev, "Debugfs create tp path failed.\n"); + goto create_path_error; + } + + if (udma_debugfs_create_files(udma_dev, dbgfs)) { + dev_err(udma_dev->dev, "Debugfs create files failed.\n"); + goto create_path_error; + } + + udma_dev->dbgfs = dbgfs; + + return; + +create_path_error: + debugfs_remove_recursive(dbgfs->root); +create_dev_error: + kfree(dbgfs); +create_error: + udma_dev->dbgfs = NULL; +} + +void udma_unregister_debugfs(struct udma_dev *udma_dev) +{ + if (IS_ERR_OR_NULL(g_udma_dbgfs_root)) + return; + + if (!udma_dev->dbgfs) + return; + + debugfs_remove_recursive(udma_dev->dbgfs->root); + kfree(udma_dev->dbgfs->private_data); + kfree(udma_dev->dbgfs); + udma_dev->dbgfs = NULL; +} + +void udma_init_debugfs(void) +{ + g_udma_dbgfs_root = debugfs_create_dir("udma", NULL); +} + +void udma_uninit_debugfs(void) +{ + debugfs_remove_recursive(g_udma_dbgfs_root); + g_udma_dbgfs_root = NULL; +} diff --git a/drivers/ub/urma/hw/udma/udma_debugfs.h b/drivers/ub/urma/hw/udma/udma_debugfs.h new file mode 100644 index 0000000000000000000000000000000000000000..d49440251dab662fdbf6a9ad3b2d96268277ada3 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_debugfs.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_DEBUGFS_H__ +#define __UDMA_DEBUGFS_H__ + +#include "udma_dev.h" +#include "udma_cmd.h" + +#define TA_MAX_SIZE 1 +#define TP_MAX_SIZE 1 +#define FILE_MOD_SIZE 2 + +enum udma_dfx_sub_opcode { + UDMA_TA_MRD, + UDMA_TP_RXTX, +}; + +enum udma_debugfs_file_mod { + RDONLY, +}; + +struct udma_debugfs_file_info { + const char *name; + enum udma_debugfs_file_mod fmod; + enum udma_cmd_opcode_type opcode; + enum udma_dfx_sub_opcode sub_opcode; +}; + +struct udma_query_rxtx_dfx { + uint32_t sub_module; + uint64_t tpp2_txdma_hdr_um_pkt_cnt; + uint64_t tpp2_txdma_ctp_rm_pkt_cnt; + uint64_t tpp2_txdma_ctp_rc_pkt_cnt; + uint64_t tpp2_txdma_tp_rm_pkt_cnt; + uint64_t tpp2_txdma_tp_rc_pkt_cnt; + uint64_t rhp_glb_rm_pkt_cnt; + uint64_t rhp_glb_rc_pkt_cnt; + uint64_t rhp_clan_rm_pkt_cnt; + uint64_t rhp_clan_rc_pkt_cnt; + uint64_t rhp_ud_pkt_cnt; + uint32_t rsvd[16]; +}; + +struct udma_query_mrd_dfx { + uint32_t sub_module; + uint32_t mrd_dsqe_issue_cnt; + uint32_t mrd_dsqe_exec_cnt; + uint32_t mrd_dsqe_drop_cnt; + uint32_t mrd_jfsdb_issue_cnt; + uint32_t mrd_jfsdb_exec_cnt; + uint32_t mrd_mb_issue_cnt; + uint32_t mrd_mb_exec_cnt; + uint32_t mrd_eqdb_issue_cnt; + uint32_t mrd_mb_buff_full; + uint32_t mrd_mb_buff_empty; + uint32_t mrd_mem_ecc_err_1b; + uint32_t mrd_mem_ecc_1b_info; + uint32_t mrd_mb_state; + uint32_t mrd_eqdb_exec_cnt; + uint32_t rsvd[7]; +}; + +struct udma_dev_debugfs { + struct dentry *root; + struct dentry *ta_root; + struct dentry *tp_root; + struct file_private_data *private_data; + uint32_t private_data_size; +}; + +struct file_private_data { + struct udma_dev *udma_dev; + enum udma_cmd_opcode_type opcode; + enum udma_dfx_sub_opcode sub_opcode; +}; + +void udma_init_debugfs(void); +void udma_uninit_debugfs(void); +void udma_unregister_debugfs(struct udma_dev *udma_dev); +void udma_register_debugfs(struct udma_dev *udma_dev); + +#endif /* __UDMA_DEBUGFS_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_def.h b/drivers/ub/urma/hw/udma/udma_def.h new file mode 100644 index 0000000000000000000000000000000000000000..0681f6dd950d7664452eb089aa27218e6d3afe2b --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_def.h @@ -0,0 +1,190 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_DEF_H__ +#define __UDMA_DEF_H__ + +#include +#include +#include +#include + +enum { + UDMA_CAP_FEATURE_AR = BIT(0), + UDMA_CAP_FEATURE_JFC_INLINE = BIT(4), + UDMA_CAP_FEATURE_DIRECT_WQE = BIT(11), + UDMA_CAP_FEATURE_CONG_CTRL = BIT(16), + UDMA_CAP_FEATURE_REDUCE = BIT(17), + UDMA_CAP_FEATURE_UE_RX_CLOSE = BIT(18), + UDMA_CAP_FEATURE_RNR_RETRY = BIT(19), +}; + +struct udma_res { + uint32_t max_cnt; + uint32_t start_idx; + uint32_t next_idx; + uint32_t depth; +}; + +struct udma_tbl { + uint32_t max_cnt; + uint32_t size; +}; + +struct udma_caps { + unsigned long init_flag; + struct udma_res jfs; + struct udma_res jfr; + struct udma_res jfc; + struct udma_res jetty; + struct udma_res jetty_grp; + uint32_t jetty_in_grp; + uint32_t jfs_sge; + uint32_t jfr_sge; + uint32_t jfs_rsge; + uint32_t jfs_inline_sz; + uint32_t comp_vector_cnt; + uint16_t ue_cnt; + uint8_t ue_id; + uint32_t trans_mode; + uint32_t max_msg_len; + uint32_t feature; + uint32_t rsvd_jetty_cnt; + uint32_t max_read_size; + uint32_t max_write_size; + uint32_t max_cas_size; + uint32_t max_fetch_and_add_size; + uint32_t atomic_feat; + struct udma_res ccu_jetty; + struct udma_res hdc_jetty; + struct udma_res stars_jetty; + struct udma_res public_jetty; + struct udma_res user_ctrl_normal_jetty; + uint16_t rc_queue_num; + uint16_t rc_queue_depth; + uint8_t rc_entry_size; + uint64_t rc_dma_len; + dma_addr_t rc_dma_addr; + uint8_t ack_queue_num; + uint8_t port_num; + uint8_t cqe_size; + struct udma_tbl seid; +}; + +struct udma_dfx_jetty { + uint32_t id; + uint32_t jfs_depth; +}; + +struct udma_dfx_jfs { + uint32_t id; + uint32_t depth; +}; + +struct udma_dfx_seg { + uint32_t id; + struct ubcore_ubva ubva; + uint64_t len; + struct ubcore_token token_value; +}; + +struct udma_dfx_entity { + uint32_t cnt; + struct xarray table; + rwlock_t rwlock; +}; + +struct udma_dfx_info { + struct udma_dfx_entity rc; + struct udma_dfx_entity jetty; + struct udma_dfx_entity jetty_grp; + struct udma_dfx_entity jfs; + struct udma_dfx_entity jfr; + struct udma_dfx_entity jfc; + struct udma_dfx_entity seg; +}; + +struct udma_sw_db_page { + struct list_head list; + struct ubcore_umem *umem; + uint64_t user_virt; + refcount_t refcount; +}; + +struct udma_hugepage_priv { + struct list_head list; + struct page **pages; + uint32_t page_num; + struct ubcore_umem *umem; + void *va_base; + uint32_t va_len; + uint32_t left_va_offset; + uint32_t left_va_len; + refcount_t refcnt; +}; + +struct udma_hugepage { + void *va_start; + uint32_t va_len; + struct udma_hugepage_priv *priv; +}; + +struct udma_buf { + dma_addr_t addr; + union { + void *kva; /* used for kernel mode */ + struct iova_slot *slot; + void *kva_or_slot; + }; + void *aligned_va; + struct ubcore_umem *umem; + uint32_t entry_size; + uint32_t entry_cnt; + uint32_t cnt_per_page_shift; + struct xarray id_table_xa; + struct mutex id_table_mutex; + bool is_hugepage; + struct udma_hugepage *hugepage; +}; + +struct udma_k_sw_db_page { + struct list_head list; + uint32_t num_db; + unsigned long *bitmap; + struct udma_buf db_buf; +}; + +struct udma_sw_db { + struct udma_sw_db_page *page; + struct udma_k_sw_db_page *kpage; + uint32_t index; + uint32_t type; + uint64_t db_addr; + uint32_t *db_record; + void *virt_addr; +}; + +struct udma_req_msg { + uint8_t dst_ue_idx; + uint8_t resp_code; + uint16_t rsv; + struct ubcore_req req; +}; + +struct udma_resp_msg { + uint8_t dst_ue_idx; + uint8_t resp_code; + uint16_t rsv; + struct ubcore_resp resp; +}; + +enum num_elem_in_grp { + NUM_TP_PER_GROUP = 16, + NUM_JETTY_PER_GROUP = 32, +}; + +enum { + RCT_INIT_FLAG, +}; + +#endif /* __UDMA_DEF_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_dev.h b/drivers/ub/urma/hw/udma/udma_dev.h new file mode 100644 index 0000000000000000000000000000000000000000..1f76ccb84c30b26840e6c7ca9b8fba7513b6267a --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_dev.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_DEV_H__ +#define __UDMA_DEV_H__ + +#include +#include +#include +#include "udma_def.h" +#include +#include + +extern bool dfx_switch; +extern bool cqe_mode; +extern uint32_t jfr_sleep_time; +extern uint32_t jfc_arm_mode; +extern bool dump_aux_info; + +#define UBCORE_MAX_DEV_NAME 64 + +#define WQE_BB_SIZE_SHIFT 6 + +#define UDMA_CTX_NUM 2 + +#define UDMA_BITS_PER_INT 32 + +#define MAX_JETTY_IN_JETTY_GRP 32 + +#define UDMA_USER_DATA_H_OFFSET 32U + +#define MAX_WQEBB_IN_SQE 4 + +#define JETTY_DSQE_OFFSET 0x1000 + +#define UDMA_HW_PAGE_SHIFT 12 +#define UDMA_HW_PAGE_SIZE (1 << UDMA_HW_PAGE_SHIFT) +#define UDMA_HUGEPAGE_SHIFT 21 +#define UDMA_HUGEPAGE_SIZE (1 << UDMA_HUGEPAGE_SHIFT) + +#define UDMA_DEV_UE_NUM 47 + +#define SEID_TABLE_SIZE 1024 + +#define UDMA_MAX_SL_NUM 16 +#define UDMA_DEFAULT_SL_NUM 0 + +#define UDMA_RCV_SEND_MAX_DIFF 512U + +#define UDMA_CQE_SIZE 64 + +#define UDMA_MAX_GRANT_SIZE 0xFFFFFFFFF000 + +enum udma_status { + UDMA_NORMAL, + UDMA_SUSPEND, +}; + +struct udma_ida { + struct ida ida; + uint32_t min; /* Lowest ID to allocate. */ + uint32_t max; /* Highest ID to allocate. */ + uint32_t next; /* Next ID to allocate. */ + spinlock_t lock; +}; + +struct udma_group_bitmap { + uint32_t min; + uint32_t max; + uint32_t grp_next; + uint32_t n_bits; + uint32_t *bit; + uint32_t bitmap_cnt; + spinlock_t lock; +}; + +struct udma_group_table { + struct xarray xa; + struct udma_group_bitmap bitmap_table; +}; + +struct udma_table { + struct xarray xa; + struct udma_ida ida_table; +}; + +struct udma_mailbox_cmd { + struct dma_pool *pool; + struct semaphore poll_sem; + struct rw_semaphore udma_mb_rwsem; +}; + +struct udma_ex_jfc_addr { + uint64_t cq_addr; + uint32_t cq_len; +}; + +struct udma_dev { + struct ubase_adev_com comdev; + struct ubcore_device ub_dev; + struct device *dev; + struct udma_caps caps; + uint16_t adev_id; + uint32_t chip_id; + uint32_t die_id; + uint32_t port_id; + uint32_t port_logic_id; + bool is_ue; + char dev_name[UBCORE_MAX_DEV_NAME]; + struct udma_mailbox_cmd mb_cmd; + struct udma_table jfr_table; + struct udma_group_table jetty_table; + struct udma_table jfc_table; + struct udma_table jetty_grp_table; + struct udma_ida rsvd_jetty_ida_table; + struct udma_table rc_table; + struct xarray crq_nb_table; + struct xarray npu_nb_table; + struct mutex npu_nb_mutex; + struct xarray ctrlq_tpid_table; + struct xarray tpn_ue_idx_table; + struct ubase_event_nb *ae_event_addr[UBASE_EVENT_TYPE_MAX]; + resource_size_t db_base; + void __iomem *k_db_base; + struct workqueue_struct *act_workq; + struct xarray ksva_table; + struct mutex ksva_mutex; + struct xarray eid_table; + struct mutex eid_mutex; + uint32_t tid; + struct iommu_sva *ksva; + struct list_head db_list[UDMA_DB_TYPE_NUM]; + struct mutex db_mutex; + struct udma_dfx_info *dfx_info; + uint32_t status; + struct udma_dev_debugfs *dbgfs; + uint32_t ue_num; + struct udma_ex_jfc_addr cq_addr_array[UDMA_JFC_TYPE_NUM]; + uint32_t ue_id; + struct page *db_page; + u8 udma_tp_sl_num; + u8 udma_ctp_sl_num; + u8 unic_sl_num; + u8 udma_total_sl_num; + u8 udma_tp_resp_vl_off; + u8 udma_tp_sl[UDMA_MAX_SL_NUM]; + u8 udma_ctp_sl[UDMA_MAX_SL_NUM]; + u8 unic_sl[UDMA_MAX_SL_NUM]; + u8 udma_sl[UDMA_MAX_SL_NUM]; + int disable_ue_rx_count; + struct mutex disable_ue_rx_mutex; + struct mutex hugepage_lock; + struct list_head hugepage_list; + uint32_t total_hugepage_num; +}; + +#define UDMA_ERR_MSG_LEN 128 +struct udma_func_map { + char err_msg[UDMA_ERR_MSG_LEN]; + int (*init_func)(struct udma_dev *udma_dev); + void (*uninit_func)(struct udma_dev *udma_dev); +}; + +static inline struct udma_dev *get_udma_dev(struct auxiliary_device *adev) +{ + return (struct udma_dev *)dev_get_drvdata(&adev->dev); +} + +static inline struct udma_dev *to_udma_dev(struct ubcore_device *ub_device) +{ + return container_of(ub_device, struct udma_dev, ub_dev); +} + +static inline void udma_id_free(struct udma_ida *ida_table, int idx) +{ + ida_free(&ida_table->ida, idx); +} + +int udma_id_alloc_auto_grow(struct udma_dev *udma_dev, struct udma_ida *ida_table, + uint32_t *idx); +int udma_id_alloc(struct udma_dev *udma_dev, struct udma_ida *ida_table, + uint32_t *idx); +int udma_adv_id_alloc(struct udma_dev *udma_dev, struct udma_group_bitmap *bitmap_table, + uint32_t *start_idx, bool is_grp, uint32_t next); +void udma_adv_id_free(struct udma_group_bitmap *bitmap_table, uint32_t start_idx, + bool is_grp); +int udma_specify_adv_id(struct udma_dev *udma_dev, struct udma_group_bitmap *bitmap_table, + uint32_t user_id); +void udma_destroy_tables(struct udma_dev *udma_dev); +int udma_init_tables(struct udma_dev *udma_dev); +int udma_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id); +void udma_remove(struct auxiliary_device *adev); +void udma_reset_init(struct auxiliary_device *adev); +void udma_reset_uninit(struct auxiliary_device *adev); +void udma_reset_down(struct auxiliary_device *adev); + +#endif /* __UDMA_DEV_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_dfx.c b/drivers/ub/urma/hw/udma/udma_dfx.c new file mode 100644 index 0000000000000000000000000000000000000000..7d186d85531d993f752be18643a82d1f86c8ca5b --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_dfx.c @@ -0,0 +1,821 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt + +#include "udma_cmd.h" +#include "udma_jfr.h" +#include "udma_jfs.h" +#include "udma_jfc.h" +#include "udma_jetty.h" +#include "udma_rct.h" +#include "udma_dfx.h" + +bool dfx_switch; + +static int to_udma_trans_mode(uint32_t type, struct udma_dev *dev, + enum ubcore_transport_mode *trans_mode) +{ + switch (type) { + case JETTY_UM: + *trans_mode = UBCORE_TP_UM; + break; + case JETTY_RC: + *trans_mode = UBCORE_TP_RC; + break; + case JETTY_RM: + *trans_mode = UBCORE_TP_RM; + break; + default: + dev_err(dev->dev, "transport mode error, type = %u.\n", type); + return -EINVAL; + } + + return 0; +} + +static int to_udma_jfr_ctx_state(uint32_t state, struct udma_dev *dev, + enum ubcore_jfr_state *jfr_state) +{ + switch (state) { + case JETTY_RESET: + *jfr_state = UBCORE_JFR_STATE_RESET; + break; + case JETTY_READY: + *jfr_state = UBCORE_JFR_STATE_READY; + break; + case JETTY_ERROR: + *jfr_state = UBCORE_JFR_STATE_ERROR; + break; + default: + dev_err(dev->dev, "JFR context state error, state = %u.\n", state); + return -EINVAL; + } + + return 0; +} + +static int to_udma_jetty_ctx_state(uint32_t state, struct udma_dev *dev, + enum ubcore_jetty_state *jetty_state) +{ + switch (state) { + case JETTY_RESET: + *jetty_state = UBCORE_JETTY_STATE_RESET; + break; + case JETTY_READY: + *jetty_state = UBCORE_JETTY_STATE_READY; + break; + case JETTY_ERROR: + *jetty_state = UBCORE_JETTY_STATE_ERROR; + break; + case JETTY_SUSPEND: + *jetty_state = UBCORE_JETTY_STATE_SUSPENDED; + break; + default: + dev_err(dev->dev, "JFS context state error, state = %u.\n", state); + return -EINVAL; + } + + return 0; +} + +int udma_query_jfr(struct ubcore_jfr *jfr, struct ubcore_jfr_cfg *cfg, + struct ubcore_jfr_attr *attr) +{ + struct udma_dev *udma_dev = to_udma_dev(jfr->ub_dev); + struct udma_jfr *udma_jfr = to_udma_jfr(jfr); + struct ubase_mbx_attr mbox_attr = {}; + struct ubase_cmd_mailbox *mailbox; + struct udma_jfr_ctx *jfr_ctx; + int ret; + + mbox_attr.tag = jfr->jfr_id.id; + mbox_attr.op = UDMA_CMD_QUERY_JFR_CONTEXT; + mailbox = udma_mailbox_query_ctx(udma_dev, &mbox_attr); + if (!mailbox) + return -ENOMEM; + + jfr_ctx = (struct udma_jfr_ctx *)mailbox->buf; + + attr->rx_threshold = to_udma_rx_threshold(jfr_ctx->limit_wl); + + ret = to_udma_jfr_ctx_state(jfr_ctx->state, udma_dev, &attr->state); + if (ret) + goto err_jfr_ctx; + + cfg->id = jfr->jfr_id.id; + cfg->flag = jfr->jfr_cfg.flag; + cfg->max_sge = 1 << jfr_ctx->rqe_size_shift; + cfg->depth = 1 << jfr_ctx->rqe_shift; + cfg->token_value.token = 0; + cfg->flag.bs.token_policy = UBCORE_TOKEN_NONE; + cfg->min_rnr_timer = jfr_ctx->rnr_timer; + + ret = to_udma_trans_mode(jfr_ctx->type, udma_dev, &cfg->trans_mode); + if (ret) + goto err_jfr_ctx; + + if (udma_jfr->rq.buf.kva) { + cfg->eid_index = jfr->jfr_cfg.eid_index; + cfg->jfc = jfr->jfr_cfg.jfc; + } + +err_jfr_ctx: + jfr_ctx->token_value = 0; + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +int udma_query_jfs(struct ubcore_jfs *jfs, struct ubcore_jfs_cfg *cfg, + struct ubcore_jfs_attr *attr) +{ + struct udma_dev *udma_dev = to_udma_dev(jfs->ub_dev); + struct udma_jfs *udma_jfs = to_udma_jfs(jfs); + struct ubase_mbx_attr mbox_attr = {}; + struct ubase_cmd_mailbox *mailbox; + struct udma_jetty_ctx *jfs_ctx; + uint32_t wqe_bb_depth; + int ret; + + mbox_attr.tag = jfs->jfs_id.id; + mbox_attr.op = UDMA_CMD_QUERY_JFS_CONTEXT; + mailbox = udma_mailbox_query_ctx(udma_dev, &mbox_attr); + if (!mailbox) + return -ENOMEM; + + jfs_ctx = (struct udma_jetty_ctx *)mailbox->buf; + + ret = to_udma_jetty_ctx_state(jfs_ctx->state, udma_dev, &attr->state); + if (ret) + goto err_jfs_ctx; + + cfg->priority = jfs_ctx->sl; + cfg->flag = jfs->jfs_cfg.flag; + cfg->max_sge = jfs->jfs_cfg.max_sge; + cfg->max_rsge = jfs->jfs_cfg.max_rsge; + cfg->err_timeout = jfs_ctx->ta_timeout; + wqe_bb_depth = 1 << jfs_ctx->sqe_bb_shift; + cfg->depth = wqe_bb_depth / udma_jfs->sq.sqe_bb_cnt; + cfg->rnr_retry = jfs_ctx->rnr_retry_num; + cfg->max_inline_data = jfs->jfs_cfg.max_inline_data; + + ret = to_udma_trans_mode(jfs_ctx->type, udma_dev, &cfg->trans_mode); + if (ret) + goto err_jfs_ctx; + + if (udma_jfs->sq.buf.kva) { + cfg->jfc = jfs->jfs_cfg.jfc; + cfg->eid_index = jfs_ctx->seid_idx; + } + +err_jfs_ctx: + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +int udma_query_jetty(struct ubcore_jetty *jetty, struct ubcore_jetty_cfg *cfg, + struct ubcore_jetty_attr *attr) +{ + struct udma_dev *udma_dev = to_udma_dev(jetty->ub_dev); + struct udma_jetty *udma_jetty = to_udma_jetty(jetty); + struct ubase_mbx_attr jfr_mbox_attr = {}; + struct ubase_cmd_mailbox *jetty_mailbox; + struct ubase_cmd_mailbox *jfr_mailbox; + struct ubase_mbx_attr mbox_attr = {}; + struct udma_jetty_ctx *jetty_ctx; + struct udma_jfr_ctx *jfr_ctx; + uint32_t wqe_bb_depth; + int ret; + + mbox_attr.tag = jetty->jetty_id.id; + mbox_attr.op = UDMA_CMD_QUERY_JFS_CONTEXT; + jetty_mailbox = udma_mailbox_query_ctx(udma_dev, &mbox_attr); + if (!jetty_mailbox) + return -ENOMEM; + + jfr_mbox_attr.tag = udma_jetty->jfr->ubcore_jfr.jfr_id.id; + jfr_mbox_attr.op = UDMA_CMD_QUERY_JFR_CONTEXT; + jfr_mailbox = udma_mailbox_query_ctx(udma_dev, &jfr_mbox_attr); + if (!jfr_mailbox) { + udma_free_cmd_mailbox(udma_dev, jetty_mailbox); + return -ENOMEM; + } + + jetty_ctx = (struct udma_jetty_ctx *)jetty_mailbox->buf; + jfr_ctx = (struct udma_jfr_ctx *)jfr_mailbox->buf; + + wqe_bb_depth = 1 << jetty_ctx->sqe_bb_shift; + cfg->id = jetty->jetty_id.id; + cfg->jfs_depth = wqe_bb_depth / udma_jetty->sq.sqe_bb_cnt; + cfg->jfr_depth = 1 << jfr_ctx->rqe_shift; + cfg->flag = jetty->jetty_cfg.flag; + cfg->max_send_sge = jetty->jetty_cfg.max_send_sge; + cfg->max_send_rsge = jetty->jetty_cfg.max_send_rsge; + cfg->max_recv_sge = jetty->jetty_cfg.max_recv_sge; + cfg->max_inline_data = jetty->jetty_cfg.max_inline_data; + cfg->priority = jetty_ctx->sl; + cfg->rnr_retry = jetty_ctx->rnr_retry_num; + cfg->err_timeout = jetty_ctx->ta_timeout; + cfg->min_rnr_timer = jetty->jetty_cfg.min_rnr_timer; + + ret = to_udma_trans_mode(jetty_ctx->type, udma_dev, &cfg->trans_mode); + if (ret) + goto err_jetty_ctx; + + cfg->token_value.token = 0; + + ret = to_udma_jetty_ctx_state(jetty_ctx->state, udma_dev, &attr->state); + if (ret) + goto err_jetty_ctx; + + attr->rx_threshold = to_udma_rx_threshold(jfr_ctx->limit_wl); + + if (udma_jetty->sq.buf.kva) { + cfg->eid_index = jetty_ctx->seid_idx; + cfg->send_jfc = jetty->jetty_cfg.send_jfc; + cfg->recv_jfc = jetty->jetty_cfg.recv_jfc; + cfg->jfr = jetty->jetty_cfg.jfr; + cfg->jetty_grp = jetty->jetty_cfg.jetty_grp; + } + +err_jetty_ctx: + jfr_ctx->token_value = 0; + udma_free_cmd_mailbox(udma_dev, jfr_mailbox); + udma_free_cmd_mailbox(udma_dev, jetty_mailbox); + + return ret; +} + +static int udma_query_res_list(struct udma_dev *udma_dev, + struct udma_dfx_entity *entity, + struct ubcore_res_val *val, + const char *name) +{ + struct ubcore_res_list_val *res_list = (struct ubcore_res_list_val *)val->addr; + size_t idx = 0; + uint32_t *id; + + res_list->cnt = 0; + + read_lock(&entity->rwlock); + if (entity->cnt == 0) { + read_unlock(&entity->rwlock); + return 0; + } + + res_list->list = vmalloc(sizeof(uint32_t) * entity->cnt); + if (!res_list->list) { + read_unlock(&entity->rwlock); + dev_err(udma_dev->dev, "failed to vmalloc %s_list, %s_cnt = %u!\n", + name, name, entity->cnt); + return -ENOMEM; + } + + xa_for_each(&entity->table, idx, id) { + if (res_list->cnt >= entity->cnt) { + read_unlock(&entity->rwlock); + vfree(res_list->list); + dev_err(udma_dev->dev, + "failed to query %s_id, %s_cnt = %u!\n", + name, name, entity->cnt); + return -EINVAL; + } + res_list->list[res_list->cnt] = idx; + ++res_list->cnt; + } + read_unlock(&entity->rwlock); + + return 0; +} + +static int udma_query_res_dev_seg(struct udma_dev *udma_dev, + struct ubcore_res_val *val) +{ + struct ubcore_res_seg_val *res_list = (struct ubcore_res_seg_val *)val->addr; + struct ubcore_seg_info *seg_list; + struct udma_dfx_seg *seg = NULL; + size_t idx; + + res_list->seg_cnt = 0; + + read_lock(&udma_dev->dfx_info->seg.rwlock); + if (udma_dev->dfx_info->seg.cnt == 0) { + read_unlock(&udma_dev->dfx_info->seg.rwlock); + return 0; + } + + seg_list = vmalloc(sizeof(*seg_list) * udma_dev->dfx_info->seg.cnt); + if (!seg_list) { + read_unlock(&udma_dev->dfx_info->seg.rwlock); + return -ENOMEM; + } + + xa_for_each(&udma_dev->dfx_info->seg.table, idx, seg) { + if (res_list->seg_cnt >= udma_dev->dfx_info->seg.cnt) { + read_unlock(&udma_dev->dfx_info->seg.rwlock); + vfree(seg_list); + dev_err(udma_dev->dev, + "failed to query seg_list, seg_cnt = %u!\n", + udma_dev->dfx_info->seg.cnt); + return -EINVAL; + } + seg_list[res_list->seg_cnt].token_id = seg->id; + seg_list[res_list->seg_cnt].len = seg->len; + seg_list[res_list->seg_cnt].ubva = seg->ubva; + seg_list[res_list->seg_cnt].ubva.va = 0; + ++res_list->seg_cnt; + } + read_unlock(&udma_dev->dfx_info->seg.rwlock); + + res_list->seg_list = seg_list; + + return 0; +} + +static int udma_query_res_rc(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_rc_val *res_rc = (struct ubcore_res_rc_val *)val->addr; + struct ubase_mbx_attr mbox_attr = {}; + struct ubase_cmd_mailbox *mailbox; + struct udma_rc_ctx *rcc; + uint32_t *rc_id; + + if (key->key_cnt == 0) + return udma_query_res_list(udma_dev, &udma_dev->dfx_info->rc, val, "rc"); + + rc_id = (uint32_t *)xa_load(&udma_dev->dfx_info->rc.table, key->key); + if (!rc_id) { + dev_err(udma_dev->dev, "failed to query rc, rc_id = %u.\n", + key->key); + return -EINVAL; + } + mbox_attr.tag = key->key; + mbox_attr.op = UDMA_CMD_QUERY_RC_CONTEXT; + mailbox = udma_mailbox_query_ctx(udma_dev, &mbox_attr); + if (!mailbox) + return -ENOMEM; + + rcc = (struct udma_rc_ctx *)mailbox->buf; + res_rc->rc_id = key->key; + res_rc->depth = 1 << rcc->rce_shift; + res_rc->type = 0; + res_rc->state = 0; + rcc->rce_base_addr_l = 0; + rcc->rce_base_addr_h = 0; + + udma_dfx_ctx_print(udma_dev, "RC", key->key, sizeof(*rcc) / sizeof(uint32_t), + (uint32_t *)rcc); + udma_free_cmd_mailbox(udma_dev, mailbox); + + return 0; +} + +static int udma_query_res_jetty(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_jetty_val *res_jetty = (struct ubcore_res_jetty_val *)val->addr; + struct ubase_mbx_attr mbox_attr = {}; + enum ubcore_jetty_state jetty_state; + struct ubase_cmd_mailbox *mailbox; + struct udma_jetty_ctx *jettyc; + struct udma_dfx_jetty *jetty; + int ret; + + if (key->key_cnt == 0) + return udma_query_res_list(udma_dev, &udma_dev->dfx_info->jetty, val, "jetty"); + + read_lock(&udma_dev->dfx_info->jetty.rwlock); + jetty = (struct udma_dfx_jetty *)xa_load(&udma_dev->dfx_info->jetty.table, key->key); + if (!jetty) { + read_unlock(&udma_dev->dfx_info->jetty.rwlock); + dev_err(udma_dev->dev, "failed to query jetty, jetty_id = %u.\n", + key->key); + return -EINVAL; + } + res_jetty->jfs_depth = jetty->jfs_depth; + read_unlock(&udma_dev->dfx_info->jetty.rwlock); + + mbox_attr.tag = key->key; + mbox_attr.op = UDMA_CMD_QUERY_JFS_CONTEXT; + mailbox = udma_mailbox_query_ctx(udma_dev, &mbox_attr); + if (!mailbox) + return -ENOMEM; + + jettyc = (struct udma_jetty_ctx *)mailbox->buf; + res_jetty->jetty_id = key->key; + + ret = to_udma_jetty_ctx_state(jettyc->state, udma_dev, &jetty_state); + if (ret) + goto err_res_jetty_ctx; + + res_jetty->state = jetty_state; + res_jetty->recv_jfc_id = jettyc->rx_jfcn; + res_jetty->send_jfc_id = jettyc->tx_jfcn; + res_jetty->priority = jettyc->sl; + res_jetty->jfr_id = jettyc->jfrn_l | + jettyc->jfrn_h << JETTY_CTX_JFRN_H_OFFSET; + jettyc->sqe_base_addr_l = 0; + jettyc->sqe_base_addr_h = 0; + jettyc->user_data_l = 0; + jettyc->user_data_h = 0; + + udma_dfx_ctx_print(udma_dev, "Jetty", key->key, sizeof(*jettyc) / sizeof(uint32_t), + (uint32_t *)jettyc); +err_res_jetty_ctx: + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +static int udma_query_res_jetty_grp(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_jetty_group_val *res_jetty_grp = + (struct ubcore_res_jetty_group_val *)val->addr; + struct udma_jetty_grp_ctx *jetty_grpc; + struct ubase_mbx_attr mbox_attr = {}; + struct ubase_cmd_mailbox *mailbox; + uint32_t *jetty_grp_id; + int i; + + if (key->key_cnt == 0) + return udma_query_res_list(udma_dev, &udma_dev->dfx_info->jetty_grp, + val, "jetty_grp"); + + jetty_grp_id = (uint32_t *)xa_load(&udma_dev->dfx_info->jetty_grp.table, key->key); + if (!jetty_grp_id) { + dev_err(udma_dev->dev, "failed to query jetty grp, jetty_grp_id = %u.\n", + key->key); + return -EINVAL; + } + + res_jetty_grp->jetty_cnt = 0; + res_jetty_grp->jetty_list = vmalloc(sizeof(*res_jetty_grp->jetty_list) * + NUM_JETTY_PER_GROUP); + if (!res_jetty_grp->jetty_list) + return -ENOMEM; + + mbox_attr.tag = key->key; + mbox_attr.op = UDMA_CMD_QUERY_JETTY_GROUP_CONTEXT; + mailbox = udma_mailbox_query_ctx(udma_dev, &mbox_attr); + if (!mailbox) { + vfree(res_jetty_grp->jetty_list); + res_jetty_grp->jetty_list = NULL; + return -ENOMEM; + } + + jetty_grpc = (struct udma_jetty_grp_ctx *)mailbox->buf; + for (i = 0; i < NUM_JETTY_PER_GROUP; ++i) { + if (jetty_grpc->valid & BIT(i)) { + res_jetty_grp->jetty_list[res_jetty_grp->jetty_cnt] = + jetty_grpc->start_jetty_id + i; + ++res_jetty_grp->jetty_cnt; + } + } + + if (res_jetty_grp->jetty_cnt == 0) { + vfree(res_jetty_grp->jetty_list); + res_jetty_grp->jetty_list = NULL; + } + + udma_dfx_ctx_print(udma_dev, "Jetty_grp", key->key, sizeof(*jetty_grpc) / sizeof(uint32_t), + (uint32_t *)jetty_grpc); + udma_free_cmd_mailbox(udma_dev, mailbox); + + return 0; +} + +static int udma_query_res_jfc(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_jfc_val *res_jfc = (struct ubcore_res_jfc_val *)val->addr; + struct ubase_mbx_attr mbox_attr = {}; + struct ubase_cmd_mailbox *mailbox; + struct udma_jfc_ctx *jfcc; + uint32_t *jfc_id; + + if (key->key_cnt == 0) + return udma_query_res_list(udma_dev, &udma_dev->dfx_info->jfc, val, "jfc"); + + jfc_id = (uint32_t *)xa_load(&udma_dev->dfx_info->jfc.table, key->key); + if (!jfc_id) { + dev_err(udma_dev->dev, "failed to query jfc, jfc_id = %u.\n", + key->key); + return -EINVAL; + } + + mbox_attr.tag = key->key; + mbox_attr.op = UDMA_CMD_QUERY_JFC_CONTEXT; + mailbox = udma_mailbox_query_ctx(udma_dev, &mbox_attr); + if (!mailbox) + return -ENOMEM; + + jfcc = (struct udma_jfc_ctx *)mailbox->buf; + res_jfc->jfc_id = key->key; + res_jfc->state = jfcc->state; + res_jfc->depth = 1 << (jfcc->shift + UDMA_JFC_DEPTH_SHIFT_BASE); + jfcc->cqe_va_l = 0; + jfcc->cqe_va_h = 0; + jfcc->token_en = 0; + jfcc->cqe_token_value = 0; + jfcc->record_db_addr_l = 0; + jfcc->record_db_addr_h = 0; + jfcc->remote_token_value = 0; + + udma_dfx_ctx_print(udma_dev, "JFC", key->key, sizeof(*jfcc) / sizeof(uint32_t), + (uint32_t *)jfcc); + udma_free_cmd_mailbox(udma_dev, mailbox); + + return 0; +} + +static int udma_query_res_jfs(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_jfs_val *res_jfs = (struct ubcore_res_jfs_val *)val->addr; + struct ubase_mbx_attr mbox_attr = {}; + enum ubcore_jetty_state jfs_state; + struct ubase_cmd_mailbox *mailbox; + struct udma_jetty_ctx *jfsc; + struct udma_dfx_jfs *jfs; + int ret; + + if (key->key_cnt == 0) + return udma_query_res_list(udma_dev, &udma_dev->dfx_info->jfs, val, "jfs"); + + read_lock(&udma_dev->dfx_info->jfs.rwlock); + jfs = (struct udma_dfx_jfs *)xa_load(&udma_dev->dfx_info->jfs.table, key->key); + if (!jfs) { + read_unlock(&udma_dev->dfx_info->jfs.rwlock); + dev_err(udma_dev->dev, "failed to query jfs, jfs_id = %u.\n", + key->key); + return -EINVAL; + } + res_jfs->depth = jfs->depth; + read_unlock(&udma_dev->dfx_info->jfs.rwlock); + + mbox_attr.tag = key->key; + mbox_attr.op = UDMA_CMD_QUERY_JFS_CONTEXT; + mailbox = udma_mailbox_query_ctx(udma_dev, &mbox_attr); + if (!mailbox) + return -ENOMEM; + + jfsc = (struct udma_jetty_ctx *)mailbox->buf; + res_jfs->jfs_id = key->key; + + ret = to_udma_jetty_ctx_state(jfsc->state, udma_dev, &jfs_state); + if (ret) + goto err_res_jetty_ctx; + + res_jfs->state = jfs_state; + res_jfs->priority = jfsc->sl; + res_jfs->jfc_id = jfsc->tx_jfcn; + jfsc->sqe_base_addr_l = 0; + jfsc->sqe_base_addr_h = 0; + jfsc->user_data_l = 0; + jfsc->user_data_h = 0; + + udma_dfx_ctx_print(udma_dev, "JFS", key->key, sizeof(*jfsc) / sizeof(uint32_t), + (uint32_t *)jfsc); +err_res_jetty_ctx: + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +static int udma_query_res_jfr(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_jfr_val *res_jfr = (struct ubcore_res_jfr_val *)val->addr; + struct ubase_mbx_attr mbox_attr = {}; + struct ubase_cmd_mailbox *mailbox; + enum ubcore_jfr_state jfr_state; + struct udma_jfr_ctx *jfrc; + uint32_t *jfr_id; + int ret; + + if (key->key_cnt == 0) + return udma_query_res_list(udma_dev, &udma_dev->dfx_info->jfr, val, "jfr"); + + jfr_id = (uint32_t *)xa_load(&udma_dev->dfx_info->jfr.table, key->key); + if (!jfr_id) { + dev_err(udma_dev->dev, "failed to query jfr, jfr_id = %u.\n", + key->key); + return -EINVAL; + } + + mbox_attr.tag = key->key; + mbox_attr.op = UDMA_CMD_QUERY_JFR_CONTEXT; + mailbox = udma_mailbox_query_ctx(udma_dev, &mbox_attr); + if (!mailbox) + return -ENOMEM; + + jfrc = (struct udma_jfr_ctx *)mailbox->buf; + res_jfr->jfr_id = key->key; + + ret = to_udma_jfr_ctx_state(jfrc->state, udma_dev, &jfr_state); + if (ret) + goto err_res_jfr_ctx; + + res_jfr->state = jfr_state; + res_jfr->depth = 1 << jfrc->rqe_shift; + res_jfr->jfc_id = jfrc->jfcn_l | + jfrc->jfcn_h << JFR_JFCN_H_OFFSET; + jfrc->rqe_base_addr_l = 0; + jfrc->rqe_base_addr_h = 0; + jfrc->token_en = 0; + jfrc->token_value = 0; + jfrc->user_data_l = 0; + jfrc->user_data_h = 0; + jfrc->idx_que_addr_l = 0; + jfrc->idx_que_addr_h = 0; + jfrc->record_db_addr_l = 0; + jfrc->record_db_addr_m = 0; + jfrc->record_db_addr_h = 0; + + udma_dfx_ctx_print(udma_dev, "JFR", key->key, sizeof(*jfrc) / sizeof(uint32_t), + (uint32_t *)jfrc); +err_res_jfr_ctx: + jfrc->token_value = 0; + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +static int udma_query_res_seg(struct udma_dev *udma_dev, struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_seg_val *res_seg = (struct ubcore_res_seg_val *)val->addr; + struct udma_dfx_seg *seg; + + if (key->key_cnt == 0) + return udma_query_res_dev_seg(udma_dev, val); + + read_lock(&udma_dev->dfx_info->seg.rwlock); + seg = (struct udma_dfx_seg *)xa_load(&udma_dev->dfx_info->seg.table, key->key); + if (!seg) { + read_unlock(&udma_dev->dfx_info->seg.rwlock); + dev_err(udma_dev->dev, "failed to query seg, token_id = %u.\n", + key->key); + return -EINVAL; + } + + res_seg->seg_list = vmalloc(sizeof(struct ubcore_seg_info)); + if (!res_seg->seg_list) { + read_unlock(&udma_dev->dfx_info->seg.rwlock); + return -ENOMEM; + } + + res_seg->seg_cnt = 1; + res_seg->seg_list->token_id = seg->id; + res_seg->seg_list->len = seg->len; + res_seg->seg_list->ubva = seg->ubva; + res_seg->seg_list->ubva.va = 0; + read_unlock(&udma_dev->dfx_info->seg.rwlock); + + return 0; +} + +static int udma_query_res_dev_ta(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_dev_ta_val *res_ta = (struct ubcore_res_dev_ta_val *)val->addr; + struct udma_dfx_info *dfx = udma_dev->dfx_info; + struct udma_dfx_entity_cnt udma_dfx_entity_cnt_ta[] = { + {&dfx->rc.rwlock, &dfx->rc, &res_ta->rc_cnt}, + {&dfx->jetty.rwlock, &dfx->jetty, &res_ta->jetty_cnt}, + {&dfx->jetty_grp.rwlock, &dfx->jetty_grp, &res_ta->jetty_group_cnt}, + {&dfx->jfs.rwlock, &dfx->jfs, &res_ta->jfs_cnt}, + {&dfx->jfr.rwlock, &dfx->jfr, &res_ta->jfr_cnt}, + {&dfx->jfc.rwlock, &dfx->jfc, &res_ta->jfc_cnt}, + {&dfx->seg.rwlock, &dfx->seg, &res_ta->seg_cnt}, + }; + + int size = ARRAY_SIZE(udma_dfx_entity_cnt_ta); + int i; + + for (i = 0; i < size; i++) { + read_lock(udma_dfx_entity_cnt_ta[i].rwlock); + *udma_dfx_entity_cnt_ta[i].res_cnt = + udma_dfx_entity_cnt_ta[i].entity->cnt; + read_unlock(udma_dfx_entity_cnt_ta[i].rwlock); + } + + return 0; +} + +typedef int (*udma_query_res_handler)(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val); + +static udma_query_res_handler g_udma_query_res_handlers[] = { + [0] = NULL, + [UBCORE_RES_KEY_VTP] = NULL, + [UBCORE_RES_KEY_TP] = NULL, + [UBCORE_RES_KEY_TPG] = NULL, + [UBCORE_RES_KEY_UTP] = NULL, + [UBCORE_RES_KEY_JFS] = udma_query_res_jfs, + [UBCORE_RES_KEY_JFR] = udma_query_res_jfr, + [UBCORE_RES_KEY_JETTY] = udma_query_res_jetty, + [UBCORE_RES_KEY_JETTY_GROUP] = udma_query_res_jetty_grp, + [UBCORE_RES_KEY_JFC] = udma_query_res_jfc, + [UBCORE_RES_KEY_RC] = udma_query_res_rc, + [UBCORE_RES_KEY_SEG] = udma_query_res_seg, + [UBCORE_RES_KEY_DEV_TA] = udma_query_res_dev_ta, + [UBCORE_RES_KEY_DEV_TP] = NULL, +}; + +int udma_query_res(struct ubcore_device *dev, struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + + if (!dfx_switch) { + dev_warn(udma_dev->dev, "the dfx_switch is not enabled.\n"); + return -EPERM; + } + + if (key->type < UBCORE_RES_KEY_VTP || key->type > UBCORE_RES_KEY_DEV_TP || + g_udma_query_res_handlers[key->type] == NULL) { + dev_err(udma_dev->dev, "key type(%u) invalid.\n", key->type); + return -EINVAL; + } + + return g_udma_query_res_handlers[key->type](udma_dev, key, val); +} + +static void list_lock_init(struct udma_dfx_info *dfx) +{ + struct udma_dfx_entity_initialization udma_dfx_entity_initialization_arr[] = { + {&dfx->rc.rwlock, &dfx->rc.table}, + {&dfx->jetty.rwlock, &dfx->jetty.table}, + {&dfx->jetty_grp.rwlock, &dfx->jetty_grp.table}, + {&dfx->jfs.rwlock, &dfx->jfs.table}, + {&dfx->jfr.rwlock, &dfx->jfr.table}, + {&dfx->jfc.rwlock, &dfx->jfc.table}, + {&dfx->seg.rwlock, &dfx->seg.table}, + }; + int size = ARRAY_SIZE(udma_dfx_entity_initialization_arr); + int i; + + for (i = 0; i < size; i++) { + rwlock_init(udma_dfx_entity_initialization_arr[i].rwlock); + xa_init(udma_dfx_entity_initialization_arr[i].table); + } +} + +int udma_dfx_init(struct udma_dev *udma_dev) +{ + if (!dfx_switch) + return 0; + + udma_dev->dfx_info = kzalloc(sizeof(struct udma_dfx_info), GFP_KERNEL); + if (!udma_dev->dfx_info) + return -ENOMEM; + + list_lock_init(udma_dev->dfx_info); + + return 0; +} + +static void udma_dfx_destroy_xa(struct udma_dev *udma_dev, struct xarray *table, + const char *name) +{ + if (!xa_empty(table)) + dev_err(udma_dev->dev, "%s table is not empty.\n", name); + xa_destroy(table); +} + +static void udma_dfx_table_free(struct udma_dev *dev) +{ + udma_dfx_destroy_xa(dev, &dev->dfx_info->rc.table, "rc"); + udma_dfx_destroy_xa(dev, &dev->dfx_info->jetty.table, "jetty"); + udma_dfx_destroy_xa(dev, &dev->dfx_info->jetty_grp.table, "jetty_grp"); + udma_dfx_destroy_xa(dev, &dev->dfx_info->jfs.table, "jfs"); + udma_dfx_destroy_xa(dev, &dev->dfx_info->jfr.table, "jfr"); + udma_dfx_destroy_xa(dev, &dev->dfx_info->jfc.table, "jfc"); + udma_dfx_destroy_xa(dev, &dev->dfx_info->seg.table, "seg"); +} + +void udma_dfx_uninit(struct udma_dev *udma_dev) +{ + if (!dfx_switch) + return; + + udma_dfx_table_free(udma_dev); + kfree(udma_dev->dfx_info); + udma_dev->dfx_info = NULL; +} + +module_param(dfx_switch, bool, 0444); +MODULE_PARM_DESC(dfx_switch, "Set whether to enable the udma_dfx function, default: 0(0:off, 1:on)"); diff --git a/drivers/ub/urma/hw/udma/udma_dfx.h b/drivers/ub/urma/hw/udma/udma_dfx.h new file mode 100644 index 0000000000000000000000000000000000000000..dcdf23646c1c3d4bfd9ed376dd7a205a539cdbeb --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_dfx.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_DFX_H__ +#define __UDMA_DFX_H__ + +#include "udma_dev.h" +#include "udma_ctx.h" +#include "udma_jetty.h" + +#define UDMA_RX_THRESHOLD_0 0 +#define UDMA_RX_THRESHOLD_64 64 +#define UDMA_RX_THRESHOLD_512 512 +#define UDMA_RX_THRESHOLD_4096 4096 + +#define UDMA_RX_REQ_EPSN_H_SHIFT 16 + +enum udma_limit_wl { + UDMA_LIMIT_WL_0, + UDMA_LIMIT_WL_64, + UDMA_LIMIT_WL_512, + UDMA_LIMIT_WL_4096, +}; + +struct udma_dfx_entity_initialization { + rwlock_t *rwlock; + struct xarray *table; +}; + +struct udma_dfx_entity_cnt { + rwlock_t *rwlock; + struct udma_dfx_entity *entity; + uint32_t *res_cnt; +}; + +static inline uint32_t to_udma_rx_threshold(uint32_t limit_wl) +{ + switch (limit_wl) { + case UDMA_LIMIT_WL_0: + return UDMA_RX_THRESHOLD_0; + case UDMA_LIMIT_WL_64: + return UDMA_RX_THRESHOLD_64; + case UDMA_LIMIT_WL_512: + return UDMA_RX_THRESHOLD_512; + default: + return UDMA_RX_THRESHOLD_4096; + } +} + +int udma_query_jfr(struct ubcore_jfr *jfr, struct ubcore_jfr_cfg *cfg, + struct ubcore_jfr_attr *attr); +int udma_query_jfs(struct ubcore_jfs *jfs, struct ubcore_jfs_cfg *cfg, + struct ubcore_jfs_attr *attr); +int udma_query_jetty(struct ubcore_jetty *jetty, struct ubcore_jetty_cfg *cfg, + struct ubcore_jetty_attr *attr); +int udma_query_res(struct ubcore_device *dev, struct ubcore_res_key *key, + struct ubcore_res_val *val); +int udma_dfx_init(struct udma_dev *udma_dev); +void udma_dfx_uninit(struct udma_dev *udma_dev); + +#endif /* __UDMA_DFX_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_eid.c b/drivers/ub/urma/hw/udma/udma_eid.c new file mode 100644 index 0000000000000000000000000000000000000000..3ff649f343be15b9a5ba2b37a565824894da740b --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_eid.c @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt + +#include +#include +#include +#include "udma_dev.h" +#include "udma_cmd.h" +#include "udma_common.h" +#include +#include "udma_eid.h" + +static void udma_dispatch_eid_event(struct udma_dev *udma_dev, + struct udma_ctrlq_eid_info *eid_entry, + enum ubcore_mgmt_event_type type) +{ + struct ubcore_mgmt_event event = {}; + struct ubcore_eid_info info = {}; + + udma_swap_endian(eid_entry->eid.raw, info.eid.raw, sizeof(union ubcore_eid)); + info.eid_index = eid_entry->eid_idx; + + event.ub_dev = &udma_dev->ub_dev; + event.element.eid_info = &info; + event.event_type = type; + ubcore_dispatch_mgmt_event(&event); +} + +int udma_add_one_eid(struct udma_dev *udma_dev, struct udma_ctrlq_eid_info *eid_info) +{ + struct udma_ctrlq_eid_info *eid_entry; + eid_t ummu_eid = 0; + guid_t guid = {}; + int ret; + + eid_entry = kzalloc(sizeof(struct udma_ctrlq_eid_info), GFP_KERNEL); + if (!eid_entry) + return -ENOMEM; + + memcpy(eid_entry, eid_info, sizeof(struct udma_ctrlq_eid_info)); + ret = xa_err(xa_store(&udma_dev->eid_table, eid_info->eid_idx, eid_entry, GFP_KERNEL)); + if (ret) { + dev_err(udma_dev->dev, + "save eid entry failed, ret = %d, eid index = %u.\n", + ret, eid_info->eid_idx); + goto store_err; + } + + if (!udma_dev->is_ue) { + (void)memcpy(&ummu_eid, eid_info->eid.raw, sizeof(ummu_eid)); + ret = ummu_core_add_eid(&guid, ummu_eid, EID_NONE); + if (ret) { + dev_err(udma_dev->dev, + "set ummu eid entry failed, ret is %d.\n", ret); + goto err_add_ummu_eid; + } + } + udma_dispatch_eid_event(udma_dev, eid_entry, UBCORE_MGMT_EVENT_EID_ADD); + + return ret; +err_add_ummu_eid: + xa_erase(&udma_dev->eid_table, eid_info->eid_idx); +store_err: + kfree(eid_entry); + + return ret; +} + +int udma_del_one_eid(struct udma_dev *udma_dev, struct udma_ctrlq_eid_info *eid_info) +{ + struct udma_ctrlq_eid_info *eid_entry; + uint32_t index = eid_info->eid_idx; + eid_t ummu_eid = 0; + guid_t guid = {}; + + eid_entry = (struct udma_ctrlq_eid_info *)xa_load(&udma_dev->eid_table, index); + if (!eid_entry) { + dev_err(udma_dev->dev, "get eid entry failed, eid index = %u.\n", + index); + return -EINVAL; + } + if (memcmp(eid_entry->eid.raw, eid_info->eid.raw, sizeof(eid_entry->eid.raw))) { + dev_err(udma_dev->dev, "eid is not match, index = %u.\n", index); + return -EINVAL; + } + xa_erase(&udma_dev->eid_table, index); + + if (!udma_dev->is_ue) { + (void)memcpy(&ummu_eid, eid_entry->eid.raw, sizeof(ummu_eid)); + ummu_core_del_eid(&guid, ummu_eid, EID_NONE); + } + udma_dispatch_eid_event(udma_dev, eid_entry, UBCORE_MGMT_EVENT_EID_RMV); + kfree(eid_entry); + + return 0; +} + +static int udma_send_query_eid_cmd(struct udma_dev *udma_dev, + struct udma_ctrlq_eid_out_query *eid_out_query) +{ +#define UDMA_CMD_CTRLQ_QUERY_SEID 0xb5 + struct udma_ctrlq_eid_in_query eid_in_query = {}; + struct ubase_ctrlq_msg msg = {}; + int ret; + + msg.opcode = UDMA_CTRLQ_GET_SEID_INFO; + msg.service_ver = UBASE_CTRLQ_SER_VER_01; + msg.service_type = UBASE_CTRLQ_SER_TYPE_DEV_REGISTER; + msg.need_resp = 1; + msg.is_resp = 0; + msg.in_size = sizeof(eid_in_query); + msg.in = &eid_in_query; + msg.out_size = sizeof(*eid_out_query); + msg.out = eid_out_query; + eid_in_query.cmd = UDMA_CMD_CTRLQ_QUERY_SEID; + + ret = ubase_ctrlq_send_msg(udma_dev->comdev.adev, &msg); + if (ret) + dev_err(udma_dev->dev, + "query seid from ctrl cpu failed, ret = %d.\n", ret); + + return ret; +} + +int udma_query_eid_from_ctrl_cpu(struct udma_dev *udma_dev) +{ + struct udma_ctrlq_eid_out_query eid_out_query = {}; + int ret, ret_tmp, i; + + ret = udma_send_query_eid_cmd(udma_dev, &eid_out_query); + if (ret) { + dev_err(udma_dev->dev, "query eid failed, ret = %d.\n", ret); + return ret; + } + + if (eid_out_query.seid_num > UDMA_CTRLQ_SEID_NUM) { + dev_err(udma_dev->dev, "Invalid param: seid num is %u.\n", eid_out_query.seid_num); + return -EINVAL; + } + + mutex_lock(&udma_dev->eid_mutex); + for (i = 0; i < (int)eid_out_query.seid_num; i++) { + if (eid_out_query.eids[i].eid_idx >= SEID_TABLE_SIZE) { + dev_err(udma_dev->dev, "invalid eid_idx = %u.\n", + eid_out_query.eids[i].eid_idx); + goto err_add_ummu_eid; + } + ret = udma_add_one_eid(udma_dev, &(eid_out_query.eids[i])); + if (ret) { + dev_err(udma_dev->dev, "Add eid failed, ret = %d, eid_idx = %u.\n", + ret, eid_out_query.eids[i].eid_idx); + goto err_add_ummu_eid; + } + } + mutex_unlock(&udma_dev->eid_mutex); + + return 0; +err_add_ummu_eid: + for (i--; i >= 0; i--) { + ret_tmp = udma_del_one_eid(udma_dev, &eid_out_query.eids[i]); + if (ret_tmp) + dev_err(udma_dev->dev, "Del eid failed, ret = %d, idx = %u.\n", + ret_tmp, eid_out_query.eids[i].eid_idx); + } + mutex_unlock(&udma_dev->eid_mutex); + + return ret; +} diff --git a/drivers/ub/urma/hw/udma/udma_eid.h b/drivers/ub/urma/hw/udma/udma_eid.h new file mode 100644 index 0000000000000000000000000000000000000000..0bb5d626503cded97414bff7289f2e84a8da5f25 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_eid.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_EID_H__ +#define __UDMA_EID_H__ + +#include +#include "udma_cmd.h" + +struct udma_seid_upi { + union ubcore_eid seid; + uint32_t upi; + uint32_t rsvd0[3]; +}; + +int udma_add_one_eid(struct udma_dev *udma_dev, struct udma_ctrlq_eid_info *eid_info); +int udma_del_one_eid(struct udma_dev *udma_dev, struct udma_ctrlq_eid_info *eid_info); +int udma_query_eid_from_ctrl_cpu(struct udma_dev *udma_dev); + +#endif /* __UDMA_EID_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_eq.c b/drivers/ub/urma/hw/udma/udma_eq.c new file mode 100644 index 0000000000000000000000000000000000000000..d3b6813b1d550d1f5fe66ce5286cbd702395cb2d --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_eq.c @@ -0,0 +1,923 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt +#define pr_fmt(fmt) "UDMA: " fmt + +#include +#include +#include +#include +#include +#include "udma_ctrlq_tp.h" +#include "udma_dev.h" +#include "udma_cmd.h" +#include "udma_jfs.h" +#include "udma_jfr.h" +#include "udma_jfc.h" +#include "udma_jetty.h" +#include "udma_eid.h" +#include +#include "udma_eq.h" + +static inline int udma_ae_tp_ctrlq_msg_deal(struct udma_dev *udma_dev, + struct ubase_aeq_notify_info *info, + uint32_t queue_num) +{ + switch (info->event_type) { + case UBASE_EVENT_TYPE_TP_FLUSH_DONE: + return udma_ctrlq_tp_flush_done(udma_dev, queue_num); + case UBASE_EVENT_TYPE_TP_LEVEL_ERROR: + return udma_ctrlq_remove_single_tp(udma_dev, queue_num, TP_ERROR); + default: + dev_warn(udma_dev->dev, "udma get unsupported async event.\n"); + return 0; + } +} + +static void dump_ae_aux_info(struct udma_dev *dev, uint8_t event_type) +{ + struct ubcore_user_ctl_out out = {}; + struct ubcore_user_ctl_in in = {}; + struct udma_ae_info_in info_in; + + if (!dump_aux_info) + return; + + info_in.event_type = event_type; + in.addr = (uint64_t)&info_in; + in.len = sizeof(struct udma_ae_info_in); + in.opcode = UDMA_USER_CTL_QUERY_AE_AUX_INFO; + + (void)udma_query_ae_aux_info(&dev->ub_dev, NULL, &in, &out); +} + +static int udma_ae_tp_level_error(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct ubase_event_nb *ev_nb = container_of(nb, struct ubase_event_nb, nb); + struct auxiliary_device *adev = (struct auxiliary_device *)ev_nb->back; + struct ubase_aeq_notify_info *info = data; + struct udma_dev *udma_dev; + uint32_t queue_num; + + queue_num = info->aeqe->event.queue_event.num; + udma_dev = get_udma_dev(adev); + + dev_warn(udma_dev->dev, + "trigger tp level ae, event type is %d, sub type is %d, queue_num is %u.\n", + info->event_type, info->sub_type, queue_num); + + return udma_ae_tp_ctrlq_msg_deal(udma_dev, info, queue_num); +} + +static int udma_ae_jfs_check_err(struct auxiliary_device *adev, uint32_t queue_num) +{ + struct udma_dev *udma_dev = get_udma_dev(adev); + struct ubcore_jetty *ubcore_jetty; + struct udma_jetty_queue *udma_sq; + struct udma_jetty *udma_jetty; + struct ubcore_jfs *ubcore_jfs; + struct udma_jfs *udma_jfs; + struct ubcore_event ae; + + xa_lock(&udma_dev->jetty_table.xa); + udma_sq = (struct udma_jetty_queue *)xa_load(&udma_dev->jetty_table.xa, queue_num); + if (!udma_sq) { + dev_warn(udma_dev->dev, + "async event for bogus queue number = %u.\n", queue_num); + xa_unlock(&udma_dev->jetty_table.xa); + return -EINVAL; + } + + if (udma_sq->is_jetty) { + udma_jetty = to_udma_jetty_from_queue(udma_sq); + ubcore_jetty = &udma_jetty->ubcore_jetty; + if (ubcore_jetty->jfae_handler) { + refcount_inc(&udma_jetty->ae_refcount); + xa_unlock(&udma_dev->jetty_table.xa); + ae.ub_dev = ubcore_jetty->ub_dev; + ae.element.jetty = ubcore_jetty; + ae.event_type = UBCORE_EVENT_JETTY_ERR; + dump_ae_aux_info(udma_dev, ae.event_type); + ubcore_jetty->jfae_handler(&ae, ubcore_jetty->uctx); + if (refcount_dec_and_test(&udma_jetty->ae_refcount)) + complete(&udma_jetty->ae_comp); + } else { + xa_unlock(&udma_dev->jetty_table.xa); + } + } else { + udma_jfs = to_udma_jfs_from_queue(udma_sq); + ubcore_jfs = &udma_jfs->ubcore_jfs; + if (ubcore_jfs->jfae_handler) { + refcount_inc(&udma_jfs->ae_refcount); + xa_unlock(&udma_dev->jetty_table.xa); + ae.ub_dev = ubcore_jfs->ub_dev; + ae.element.jfs = ubcore_jfs; + ae.event_type = UBCORE_EVENT_JFS_ERR; + dump_ae_aux_info(udma_dev, ae.event_type); + ubcore_jfs->jfae_handler(&ae, ubcore_jfs->uctx); + if (refcount_dec_and_test(&udma_jfs->ae_refcount)) + complete(&udma_jfs->ae_comp); + } else { + xa_unlock(&udma_dev->jetty_table.xa); + } + } + + return 0; +} + +static int udma_ae_jfr_check_err(struct auxiliary_device *adev, uint32_t queue_num, + enum ubcore_event_type ubcore_etype) +{ + struct udma_dev *udma_dev = get_udma_dev(adev); + struct ubcore_jfr *ubcore_jfr; + struct udma_jfr *udma_jfr; + struct ubcore_event ae; + + xa_lock(&udma_dev->jfr_table.xa); + udma_jfr = (struct udma_jfr *)xa_load(&udma_dev->jfr_table.xa, queue_num); + if (!udma_jfr) { + dev_warn(udma_dev->dev, + "async event for bogus jfr number = %u.\n", queue_num); + xa_unlock(&udma_dev->jfr_table.xa); + return -EINVAL; + } + + ubcore_jfr = &udma_jfr->ubcore_jfr; + if (ubcore_jfr->jfae_handler) { + refcount_inc(&udma_jfr->ae_refcount); + xa_unlock(&udma_dev->jfr_table.xa); + ae.ub_dev = ubcore_jfr->ub_dev; + ae.element.jfr = ubcore_jfr; + ae.event_type = ubcore_etype; + ubcore_jfr->jfae_handler(&ae, ubcore_jfr->uctx); + if (refcount_dec_and_test(&udma_jfr->ae_refcount)) + complete(&udma_jfr->ae_comp); + } else { + xa_unlock(&udma_dev->jfr_table.xa); + } + + return 0; +} + +static int udma_ae_jfc_check_err(struct auxiliary_device *adev, uint32_t queue_num) +{ + struct udma_dev *udma_dev = get_udma_dev(adev); + struct ubcore_jfc *ubcore_jfc; + struct udma_jfc *udma_jfc; + struct ubcore_event ae; + unsigned long flags; + + xa_lock_irqsave(&udma_dev->jfc_table.xa, flags); + udma_jfc = (struct udma_jfc *)xa_load(&udma_dev->jfc_table.xa, queue_num); + if (!udma_jfc) { + dev_warn(udma_dev->dev, + "async event for bogus jfc number = %u.\n", queue_num); + xa_unlock_irqrestore(&udma_dev->jfc_table.xa, flags); + return -EINVAL; + } + + ubcore_jfc = &udma_jfc->base; + if (ubcore_jfc->jfae_handler) { + refcount_inc(&udma_jfc->event_refcount); + xa_unlock_irqrestore(&udma_dev->jfc_table.xa, flags); + ae.ub_dev = ubcore_jfc->ub_dev; + ae.element.jfc = ubcore_jfc; + ae.event_type = UBCORE_EVENT_JFC_ERR; + dump_ae_aux_info(udma_dev, ae.event_type); + ubcore_jfc->jfae_handler(&ae, ubcore_jfc->uctx); + if (refcount_dec_and_test(&udma_jfc->event_refcount)) + complete(&udma_jfc->event_comp); + } else { + xa_unlock_irqrestore(&udma_dev->jfc_table.xa, flags); + } + + return 0; +} + +static int udma_ae_jetty_group_check_err(struct auxiliary_device *adev, uint32_t queue_num) +{ + struct udma_dev *udma_dev = get_udma_dev(adev); + struct ubcore_jetty_group *ubcore_jetty_grp; + struct udma_jetty_grp *udma_jetty_grp; + struct ubcore_event ae; + + xa_lock(&udma_dev->jetty_grp_table.xa); + udma_jetty_grp = (struct udma_jetty_grp *)xa_load(&udma_dev->jetty_grp_table.xa, queue_num); + if (!udma_jetty_grp) { + dev_warn(udma_dev->dev, + "async event for bogus jetty group number = %u.\n", queue_num); + xa_unlock(&udma_dev->jetty_grp_table.xa); + return -EINVAL; + } + + ubcore_jetty_grp = &udma_jetty_grp->ubcore_jetty_grp; + if (ubcore_jetty_grp->jfae_handler) { + refcount_inc(&udma_jetty_grp->ae_refcount); + xa_unlock(&udma_dev->jetty_grp_table.xa); + ae.ub_dev = ubcore_jetty_grp->ub_dev; + ae.element.jetty_grp = ubcore_jetty_grp; + ae.event_type = UBCORE_EVENT_JETTY_GRP_ERR; + ubcore_jetty_grp->jfae_handler(&ae, ubcore_jetty_grp->uctx); + if (refcount_dec_and_test(&udma_jetty_grp->ae_refcount)) + complete(&udma_jetty_grp->ae_comp); + } else { + xa_unlock(&udma_dev->jetty_grp_table.xa); + } + + return 0; +} + +static int udma_ae_jetty_level_error(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct ubase_event_nb *ev_nb = container_of(nb, struct ubase_event_nb, nb); + struct auxiliary_device *adev = (struct auxiliary_device *)ev_nb->back; + struct ubase_aeq_notify_info *info = data; + uint32_t queue_num; + + queue_num = info->aeqe->event.queue_event.num; + + dev_warn(&adev->dev, + "trigger jetty level ae, event type is %d, sub type is %d, queue_num is %u.\n", + info->event_type, info->sub_type, queue_num); + + if (info->event_type == UBASE_EVENT_TYPE_JFR_LIMIT_REACHED) + return udma_ae_jfr_check_err(adev, queue_num, UBCORE_EVENT_JFR_LIMIT_REACHED); + + switch (info->sub_type) { + case UBASE_SUBEVENT_TYPE_JFS_CHECK_ERROR: + return udma_ae_jfs_check_err(adev, queue_num); + case UBASE_SUBEVENT_TYPE_JFR_CHECK_ERROR: + return udma_ae_jfr_check_err(adev, queue_num, UBCORE_EVENT_JFR_ERR); + case UBASE_SUBEVENT_TYPE_JFC_CHECK_ERROR: + return udma_ae_jfc_check_err(adev, queue_num); + case UBASE_SUBEVENT_TYPE_JETTY_GROUP_CHECK_ERROR: + return udma_ae_jetty_group_check_err(adev, queue_num); + default: + dev_warn(&adev->dev, + "udma get unsupported async event.\n"); + return -EINVAL; + } +} + +struct ae_operation { + uint32_t op_code; + notifier_fn_t call; +}; + +static struct ae_operation udma_ae_opts[] = { + {UBASE_EVENT_TYPE_JETTY_LEVEL_ERROR, udma_ae_jetty_level_error}, + {UBASE_EVENT_TYPE_JFR_LIMIT_REACHED, udma_ae_jetty_level_error}, + {UBASE_EVENT_TYPE_TP_LEVEL_ERROR, udma_ae_tp_level_error}, + {UBASE_EVENT_TYPE_TP_FLUSH_DONE, udma_ae_tp_level_error}, +}; + +void udma_unregister_ae_event(struct auxiliary_device *adev) +{ + struct udma_dev *udma_dev = get_udma_dev(adev); + int i; + + for (i = 0; i < UBASE_EVENT_TYPE_MAX; i++) { + if (udma_dev->ae_event_addr[i]) { + ubase_event_unregister(adev, udma_dev->ae_event_addr[i]); + kfree(udma_dev->ae_event_addr[i]); + udma_dev->ae_event_addr[i] = NULL; + } + } +} + +static int +udma_event_register(struct auxiliary_device *adev, enum ubase_event_type event_type, + int (*call)(struct notifier_block *nb, + unsigned long action, void *data)) +{ + struct udma_dev *udma_dev = get_udma_dev(adev); + struct ubase_event_nb *cb; + int ret = 0; + + cb = kzalloc(sizeof(*cb), GFP_KERNEL); + if (!cb) + return -ENOMEM; + + cb->drv_type = UBASE_DRV_UDMA; + cb->event_type = event_type; + cb->back = (void *)adev; + cb->nb.notifier_call = call; + + ret = ubase_event_register(adev, cb); + if (ret) { + dev_err(&adev->dev, + "failed to register async event, event type = %u, ret = %d.\n", + cb->event_type, ret); + kfree(cb); + return ret; + } + udma_dev->ae_event_addr[event_type] = cb; + + return 0; +} + +/* thanks to drivers/infiniband/hw/erdma/erdma_eq.c */ +int udma_register_ae_event(struct auxiliary_device *adev) +{ + uint32_t i, opt_num; + int ret; + + opt_num = sizeof(udma_ae_opts) / sizeof(struct ae_operation); + for (i = 0; i < opt_num; ++i) { + ret = udma_event_register(adev, udma_ae_opts[i].op_code, udma_ae_opts[i].call); + if (ret) { + udma_unregister_ae_event(adev); + break; + } + } + + return ret; +} + +/* thanks to drivers/infiniband/hw/erdma/erdma_eq.c */ +int udma_register_ce_event(struct auxiliary_device *adev) +{ + int ret; + + ret = ubase_comp_register(adev, udma_jfc_completion); + if (ret) + dev_err(&adev->dev, + "failed to register ce event, ret: %d.\n", ret); + + return ret; +} + +static inline bool udma_check_tpn_ue_idx(struct udma_ue_idx_table *tp_ue_idx_info, + uint8_t ue_idx) +{ + int i; + + for (i = 0; i < tp_ue_idx_info->num; i++) { + if (tp_ue_idx_info->ue_idx[i] == ue_idx) + return true; + } + + return false; +} + +static int udma_save_tpn_ue_idx_info(struct udma_dev *udma_dev, uint8_t ue_idx, + uint32_t tpn) +{ + struct udma_ue_idx_table *tp_ue_idx_info; + int ret; + + xa_lock(&udma_dev->tpn_ue_idx_table); + tp_ue_idx_info = xa_load(&udma_dev->tpn_ue_idx_table, tpn); + if (tp_ue_idx_info) { + if (tp_ue_idx_info->num >= UDMA_UE_NUM) { + dev_err(udma_dev->dev, + "num exceeds the maximum value.\n"); + xa_unlock(&udma_dev->tpn_ue_idx_table); + + return -EINVAL; + } + + if (!udma_check_tpn_ue_idx(tp_ue_idx_info, ue_idx)) + tp_ue_idx_info->ue_idx[tp_ue_idx_info->num++] = ue_idx; + + xa_unlock(&udma_dev->tpn_ue_idx_table); + + return 0; + } + xa_unlock(&udma_dev->tpn_ue_idx_table); + + tp_ue_idx_info = kzalloc(sizeof(*tp_ue_idx_info), GFP_KERNEL); + if (!tp_ue_idx_info) + return -ENOMEM; + + tp_ue_idx_info->ue_idx[tp_ue_idx_info->num++] = ue_idx; + ret = xa_err(xa_store(&udma_dev->tpn_ue_idx_table, tpn, tp_ue_idx_info, + GFP_KERNEL)); + if (ret) { + dev_err(udma_dev->dev, + "store tpn ue idx table failed, ret is %d.\n", ret); + goto err_store_ue_id; + } + + return ret; + +err_store_ue_id: + kfree(tp_ue_idx_info); + return ret; +} + +static void udma_delete_tpn_ue_idx_info(struct udma_dev *udma_dev, uint32_t tpn) +{ + struct udma_ue_idx_table *tp_ue_idx_info; + + xa_lock(&udma_dev->tpn_ue_idx_table); + tp_ue_idx_info = xa_load(&udma_dev->tpn_ue_idx_table, tpn); + if (tp_ue_idx_info) { + tp_ue_idx_info->num--; + if (tp_ue_idx_info->num == 0) { + __xa_erase(&udma_dev->tpn_ue_idx_table, tpn); + kfree(tp_ue_idx_info); + } + } + xa_unlock(&udma_dev->tpn_ue_idx_table); +} + +static int udma_save_tp_info(struct udma_dev *udma_dev, struct udma_ue_tp_info *info, + uint8_t ue_idx) +{ +#define UDMA_RSP_TP_MUL 2 + uint32_t tpn; + int ret = 0; + int i; + + for (i = 0; i < info->tp_cnt * UDMA_RSP_TP_MUL; i++) { + tpn = info->start_tpn + i; + ret = udma_save_tpn_ue_idx_info(udma_dev, ue_idx, tpn); + if (ret) { + dev_err(udma_dev->dev, "save tpn info fail, ret = %d, tpn = %u.\n", + ret, tpn); + goto err_save_ue_id; + } + } + + return ret; + +err_save_ue_id: + for (i--; i >= 0; i--) { + tpn = info->start_tpn + i; + udma_delete_tpn_ue_idx_info(udma_dev, tpn); + } + + return ret; +} + +static int udma_crq_recv_req_msg(void *dev, void *data, uint32_t len) +{ + struct udma_dev *udma_dev = get_udma_dev((struct auxiliary_device *)dev); + struct udma_ue_tp_info *info; + struct udma_req_msg *req; + + if (len < sizeof(*req) + sizeof(*info)) { + dev_err(udma_dev->dev, "len of crq req is too small, len = %u.\n", len); + return -EINVAL; + } + req = (struct udma_req_msg *)data; + + if (req->resp_code != UDMA_CMD_NOTIFY_MUE_SAVE_TP) { + dev_err(udma_dev->dev, "ue to mue opcode error, opcode = %u.\n", + req->resp_code); + return -EINVAL; + } + info = (struct udma_ue_tp_info *)req->req.data; + + return udma_save_tp_info(udma_dev, info, req->dst_ue_idx); +} + +static void udma_activate_dev_work(struct work_struct *work) +{ + struct udma_flush_work *flush_work = container_of(work, struct udma_flush_work, work); + struct udma_dev *udev = flush_work->udev; + int ret; + + ret = udma_open_ue_rx(udev, true, false, false, 0); + if (ret) + dev_err(udev->dev, "udma open ue rx failed, ret = %d.\n", ret); + + kfree(flush_work); +} + +static int udma_crq_recv_resp_msg(void *dev, void *data, uint32_t len) +{ + struct udma_dev *udma_dev = get_udma_dev((struct auxiliary_device *)dev); + struct udma_flush_work *flush_work; + struct udma_resp_msg *udma_resp; + + if (len < sizeof(*udma_resp)) { + dev_err(udma_dev->dev, "len of crq resp is too small, len = %u.\n", len); + return -EINVAL; + } + udma_resp = (struct udma_resp_msg *)data; + if (udma_resp->resp_code != UDMA_CMD_NOTIFY_UE_FLUSH_DONE) { + dev_err(udma_dev->dev, "mue to ue opcode err, opcode = %u.\n", + udma_resp->resp_code); + return -EINVAL; + } + + flush_work = kzalloc(sizeof(*flush_work), GFP_ATOMIC); + if (!flush_work) + return -ENOMEM; + + flush_work->udev = udma_dev; + INIT_WORK(&flush_work->work, udma_activate_dev_work); + queue_work(udma_dev->act_workq, &flush_work->work); + + return 0; +} + +static struct ubase_crq_event_nb udma_crq_opts[] = { + {UBASE_OPC_UE_TO_MUE, NULL, udma_crq_recv_req_msg}, + {UBASE_OPC_MUE_TO_UE, NULL, udma_crq_recv_resp_msg}, +}; + +void udma_unregister_crq_event(struct auxiliary_device *adev) +{ + struct udma_dev *udma_dev = get_udma_dev(adev); + struct ubase_crq_event_nb *nb = NULL; + size_t index; + + xa_for_each(&udma_dev->crq_nb_table, index, nb) { + xa_erase(&udma_dev->crq_nb_table, index); + ubase_unregister_crq_event(adev, nb->opcode); + kfree(nb); + nb = NULL; + } +} + +static int udma_register_one_crq_event(struct auxiliary_device *adev, + struct ubase_crq_event_nb *crq_nb, + uint32_t index) +{ + struct udma_dev *udma_dev = get_udma_dev(adev); + struct ubase_crq_event_nb *nb; + int ret; + + nb = kzalloc(sizeof(*nb), GFP_KERNEL); + if (!nb) + return -ENOMEM; + + nb->opcode = crq_nb->opcode; + nb->back = adev; + nb->crq_handler = crq_nb->crq_handler; + ret = ubase_register_crq_event(adev, nb); + if (ret) { + dev_err(udma_dev->dev, + "register crq event failed, opcode is %u, ret is %d.\n", + nb->opcode, ret); + goto err_register_crq_event; + } + + ret = xa_err(xa_store(&udma_dev->crq_nb_table, index, nb, GFP_KERNEL)); + if (ret) { + dev_err(udma_dev->dev, + "save crq nb entry failed, opcode is %u, ret is %d.\n", + nb->opcode, ret); + goto err_store_crq_nb; + } + + return ret; + +err_store_crq_nb: + ubase_unregister_crq_event(adev, nb->opcode); +err_register_crq_event: + kfree(nb); + return ret; +} + +int udma_register_crq_event(struct auxiliary_device *adev) +{ + uint32_t opt_num = sizeof(udma_crq_opts) / sizeof(struct ubase_crq_event_nb); + uint32_t index; + int ret = 0; + + for (index = 0; index < opt_num; ++index) { + ret = udma_register_one_crq_event(adev, &udma_crq_opts[index], index); + if (ret) { + udma_unregister_crq_event(adev); + break; + } + } + + return ret; +} + +static int udma_ctrlq_send_eid_update_response(struct udma_dev *udma_dev, uint16_t seq, int ret_val) +{ + struct ubase_ctrlq_msg msg = {}; + int inbuf = 0; + int ret; + + msg.service_ver = UBASE_CTRLQ_SER_VER_01; + msg.service_type = UBASE_CTRLQ_SER_TYPE_DEV_REGISTER; + msg.opcode = UDMA_CTRLQ_UPDATE_SEID_INFO; + msg.need_resp = 0; + msg.is_resp = 1; + msg.resp_seq = seq; + msg.resp_ret = (uint8_t)(-ret_val); + msg.in = (void *)&inbuf; + msg.in_size = sizeof(inbuf); + + ret = ubase_ctrlq_send_msg(udma_dev->comdev.adev, &msg); + if (ret) + dev_err(udma_dev->dev, "send eid update response failed, ret = %d, ret_val = %d.\n", + ret, ret_val); + return ret; +} + +static int udma_ctrlq_eid_update(struct auxiliary_device *adev, uint8_t service_ver, + void *data, uint16_t len, uint16_t seq) +{ + struct udma_ctrlq_eid_out_update eid_entry = {}; + struct udma_dev *udma_dev; + int ret; + + if (adev == NULL || data == NULL) { + pr_err("adev or data is NULL.\n"); + return -EINVAL; + } + + udma_dev = get_udma_dev(adev); + if (udma_dev->status != UDMA_NORMAL) + return udma_ctrlq_send_eid_update_response(udma_dev, seq, 0); + + if (len < sizeof(struct udma_ctrlq_eid_out_update)) { + dev_err(udma_dev->dev, "msg len(%u) is invalid.\n", len); + return udma_ctrlq_send_eid_update_response(udma_dev, seq, -EINVAL); + } + memcpy(&eid_entry, data, sizeof(eid_entry)); + if (eid_entry.op_type != UDMA_CTRLQ_EID_ADD && eid_entry.op_type != UDMA_CTRLQ_EID_DEL) { + dev_err(udma_dev->dev, "update eid op type(%u) is invalid.\n", eid_entry.op_type); + return udma_ctrlq_send_eid_update_response(udma_dev, seq, -EINVAL); + } + if (eid_entry.eid_info.eid_idx >= SEID_TABLE_SIZE) { + dev_err(udma_dev->dev, "update invalid eid_idx = %u.\n", + eid_entry.eid_info.eid_idx); + return udma_ctrlq_send_eid_update_response(udma_dev, seq, -EINVAL); + } + mutex_lock(&udma_dev->eid_mutex); + if (eid_entry.op_type == UDMA_CTRLQ_EID_ADD) + ret = udma_add_one_eid(udma_dev, &(eid_entry.eid_info)); + else + ret = udma_del_one_eid(udma_dev, &(eid_entry.eid_info)); + if (ret) + dev_err(udma_dev->dev, "update eid failed, op = %u, index = %u, ret = %d.\n", + eid_entry.op_type, eid_entry.eid_info.eid_idx, ret); + mutex_unlock(&udma_dev->eid_mutex); + + return udma_ctrlq_send_eid_update_response(udma_dev, seq, ret); +} + +static int udma_ctrlq_check_tp_status(struct udma_dev *udev, void *data, uint16_t len, + struct udma_ctrlq_check_tp_active_rsp_info **rsp_info, + uint32_t *rsp_info_len) +{ +#define UDMA_CTRLQ_CHECK_TP_OFFSET 0xFF + struct udma_ctrlq_check_tp_active_req_info *req_info = NULL; + uint32_t req_info_len; + uint32_t tp_num; + int i; + + tp_num = *((uint32_t *)data) & UDMA_CTRLQ_CHECK_TP_OFFSET; + req_info_len = sizeof(struct udma_ctrlq_check_tp_active_req_info) + + sizeof(struct udma_ctrlq_check_tp_active_req_data) * tp_num; + if (len < req_info_len) { + dev_err(udev->dev, "msg param num(%u) is invalid.\n", tp_num); + return -EINVAL; + } + req_info = kzalloc(req_info_len, GFP_KERNEL); + if (!req_info) + return -ENOMEM; + memcpy(req_info, data, req_info_len); + + *rsp_info_len = sizeof(struct udma_ctrlq_check_tp_active_rsp_info) + + sizeof(struct udma_ctrlq_check_tp_active_rsp_data) * tp_num; + *rsp_info = kzalloc(*rsp_info_len, GFP_KERNEL); + if (!(*rsp_info)) { + *rsp_info_len = 0; + kfree(req_info); + req_info = NULL; + return -ENOMEM; + } + + rcu_read_lock(); + for (i = 0; i < req_info->num; i++) { + if (find_vpid(req_info->data[i].pid_flag)) + (*rsp_info)->data[i].result = UDMA_CTRLQ_TPID_IN_USE; + else + (*rsp_info)->data[i].result = UDMA_CTRLQ_TPID_EXITED; + + (*rsp_info)->data[i].tp_id = req_info->data[i].tp_id; + } + (*rsp_info)->num = tp_num; + rcu_read_unlock(); + + if (debug_switch) + udma_dfx_ctx_print(udev, "udma check tp active", (*rsp_info)->data[0].tp_id, + *rsp_info_len / sizeof(uint32_t), (uint32_t *)(*rsp_info)); + kfree(req_info); + req_info = NULL; + + return 0; +} + +static int udma_ctrlq_check_tp_active_param(struct udma_dev *udev, void *data, uint16_t len) +{ + if (data == NULL) { + dev_err(udev->dev, "data is NULL.\n"); + return -EINVAL; + } + + if (len < sizeof(struct udma_ctrlq_check_tp_active_req_info)) { + dev_err(udev->dev, "msg data len(%u) is invalid.\n", len); + return -EINVAL; + } + + return 0; +} + +static int udma_ctrlq_check_tp_active(struct auxiliary_device *adev, + uint8_t service_ver, void *data, + uint16_t len, uint16_t seq) +{ + struct udma_ctrlq_check_tp_active_rsp_info *rsp_info = NULL; + struct udma_dev *udev = get_udma_dev(adev); + struct ubase_ctrlq_msg msg = {}; + uint32_t rsp_info_len = 0; + int ret; + + ret = udma_ctrlq_check_tp_active_param(udev, data, len); + if (ret == 0) { + ret = udma_ctrlq_check_tp_status(udev, data, len, &rsp_info, &rsp_info_len); + if (ret) + dev_err(udev->dev, "check tp status failed, ret(%d).\n", ret); + } + + msg.service_ver = UBASE_CTRLQ_SER_VER_01; + msg.service_type = UBASE_CTRLQ_SER_TYPE_TP_ACL; + msg.opcode = UDMA_CMD_CTRLQ_CHECK_TP_ACTIVE; + msg.need_resp = 0; + msg.is_resp = 1; + msg.in_size = (uint16_t)rsp_info_len; + msg.in = (void *)rsp_info; + msg.resp_seq = seq; + msg.resp_ret = (uint8_t)(-ret); + + ret = ubase_ctrlq_send_msg(adev, &msg); + if (ret) + dev_err(udev->dev, "send check tp active ctrlq msg failed, ret(%d).\n", ret); + + kfree(rsp_info); + rsp_info = NULL; + + return ret; +} + +static int udma_ctrlq_send_eid_guid_response(struct udma_dev *udma_dev, + uint16_t seq, + int ret_val) +{ + struct ubase_ctrlq_msg msg = {}; + int in_buf = 0; + int ret; + + msg.service_ver = UBASE_CTRLQ_SER_VER_01; + msg.service_type = UBASE_CTRLQ_SER_TYPE_DEV_REGISTER; + msg.opcode = UDMA_CTRLQ_OPC_UPDATE_UE_SEID_GUID; + msg.need_resp = 0; + msg.is_resp = 1; + msg.resp_seq = seq; + msg.resp_ret = (uint8_t)(-ret_val); + msg.in = (void *)&in_buf; + msg.in_size = sizeof(in_buf); + + ret = ubase_ctrlq_send_msg(udma_dev->comdev.adev, &msg); + if (ret) + dev_err(udma_dev->dev, "send eid-guid rsp failed, ret = %d.\n", + ret); + + return ret; +} + +static int udma_ctrlq_notify_mue_eid_guid(struct auxiliary_device *adev, + uint8_t service_ver, + void *data, + uint16_t len, + uint16_t seq) +{ + struct udma_ctrlq_ue_eid_guid_out eid_guid_entry = {}; + struct udma_dev *udma_dev; + + if (adev == NULL || data == NULL) { + pr_err("adev is null : %d, data is null : %d.\n", + adev == NULL, data == NULL); + return -EINVAL; + } + + udma_dev = get_udma_dev(adev); + if (udma_dev->is_ue) + return 0; + + if (udma_dev->status != UDMA_NORMAL) + return udma_ctrlq_send_eid_guid_response(udma_dev, seq, 0); + if (len < sizeof(struct udma_ctrlq_ue_eid_guid_out)) { + dev_err(udma_dev->dev, "eid-guid len(%u) is invalid.\n", len); + return udma_ctrlq_send_eid_guid_response(udma_dev, seq, -EINVAL); + } + memcpy(&eid_guid_entry, data, sizeof(eid_guid_entry)); + if (eid_guid_entry.op_type != UDMA_CTRLQ_EID_GUID_ADD && + eid_guid_entry.op_type != UDMA_CTRLQ_EID_GUID_DEL) { + dev_err(udma_dev->dev, "eid-guid type(%u) is invalid.\n", + eid_guid_entry.op_type); + return udma_ctrlq_send_eid_guid_response(udma_dev, seq, + -EINVAL); + } + if (eid_guid_entry.eid_info.eid_idx >= SEID_TABLE_SIZE) { + dev_err(udma_dev->dev, "invalid ue eid_idx = %u.\n", + eid_guid_entry.eid_info.eid_idx); + return udma_ctrlq_send_eid_guid_response(udma_dev, seq, + -EINVAL); + } + + return udma_ctrlq_send_eid_guid_response(udma_dev, seq, 0); +} + +static struct ubase_ctrlq_event_nb udma_ctrlq_opts[] = { + {UBASE_CTRLQ_SER_TYPE_TP_ACL, UDMA_CMD_CTRLQ_CHECK_TP_ACTIVE, NULL, + udma_ctrlq_check_tp_active}, + {UBASE_CTRLQ_SER_TYPE_DEV_REGISTER, UDMA_CTRLQ_UPDATE_SEID_INFO, NULL, + udma_ctrlq_eid_update}, + {UBASE_CTRLQ_SER_TYPE_DEV_REGISTER, UDMA_CTRLQ_OPC_UPDATE_UE_SEID_GUID, NULL, + udma_ctrlq_notify_mue_eid_guid}, +}; + +static int udma_register_one_ctrlq_event(struct auxiliary_device *adev, + struct ubase_ctrlq_event_nb *ctrlq_nb, + uint32_t index) +{ + struct udma_dev *udma_dev = get_udma_dev(adev); + struct ubase_ctrlq_event_nb *nb; + int ret; + + nb = kzalloc(sizeof(*nb), GFP_KERNEL); + if (nb == NULL) + return -ENOMEM; + + nb->service_type = ctrlq_nb->service_type; + nb->opcode = ctrlq_nb->opcode; + nb->back = adev; + nb->crq_handler = ctrlq_nb->crq_handler; + ret = ubase_ctrlq_register_crq_event(adev, nb); + if (ret) + dev_err(udma_dev->dev, + "ubase register ctrlq event failed, opcode = %u, ret is %d.\n", + nb->opcode, ret); + + kfree(nb); + + return ret; +} + +void udma_unregister_ctrlq_event(struct auxiliary_device *adev) +{ + int opt_num; + int index; + + opt_num = ARRAY_SIZE(udma_ctrlq_opts); + for (index = 0; index < opt_num; ++index) + ubase_ctrlq_unregister_crq_event(adev, udma_ctrlq_opts[index].service_type, + udma_ctrlq_opts[index].opcode); +} + +int udma_register_ctrlq_event(struct auxiliary_device *adev) +{ + int opt_num; + int index; + int ret; + + opt_num = ARRAY_SIZE(udma_ctrlq_opts); + for (index = 0; index < opt_num; ++index) { + ret = udma_register_one_ctrlq_event(adev, &udma_ctrlq_opts[index], index); + if (ret) + goto err_register_one_ctrlq_event; + } + + return ret; + +err_register_one_ctrlq_event: + for (index--; index >= 0; index--) { + ubase_ctrlq_unregister_crq_event(adev, + udma_ctrlq_opts[index].service_type, + udma_ctrlq_opts[index].opcode); + } + + return ret; +} + +int udma_register_activate_workqueue(struct udma_dev *udma_dev) +{ + udma_dev->act_workq = alloc_workqueue("udma_activate_workq", WQ_UNBOUND, 0); + if (!udma_dev->act_workq) { + dev_err(udma_dev->dev, "failed to create activate workqueue.\n"); + return -ENOMEM; + } + + return 0; +} + +void udma_unregister_activate_workqueue(struct udma_dev *udma_dev) +{ + flush_workqueue(udma_dev->act_workq); + destroy_workqueue(udma_dev->act_workq); +} diff --git a/drivers/ub/urma/hw/udma/udma_eq.h b/drivers/ub/urma/hw/udma/udma_eq.h new file mode 100644 index 0000000000000000000000000000000000000000..c0d727de78b6b9a443f2b17269325e3bf87f2f16 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_eq.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_EQ_H__ +#define __UDMA_EQ_H__ + +int udma_register_ae_event(struct auxiliary_device *adev); +void udma_unregister_ae_event(struct auxiliary_device *adev); +int udma_register_ce_event(struct auxiliary_device *adev); +void udma_unregister_crq_event(struct auxiliary_device *adev); +int udma_register_crq_event(struct auxiliary_device *adev); +int udma_register_ctrlq_event(struct auxiliary_device *adev); +void udma_unregister_ctrlq_event(struct auxiliary_device *adev); +int udma_register_activate_workqueue(struct udma_dev *udma_dev); +void udma_unregister_activate_workqueue(struct udma_dev *udma_dev); + +static inline void udma_unregister_ce_event(struct auxiliary_device *adev) +{ + ubase_comp_unregister(adev); +} + +struct udma_flush_work { + struct udma_dev *udev; + struct work_struct work; +}; + +#endif /* __UDMA_EQ_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_jetty.c b/drivers/ub/urma/hw/udma/udma_jetty.c new file mode 100644 index 0000000000000000000000000000000000000000..e41b85e710544ad36b8942af3284040c654c259e --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_jetty.c @@ -0,0 +1,1699 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt +#define pr_fmt(fmt) "UDMA: " fmt + +#include +#include +#include +#include +#include "udma_dev.h" +#include +#include "udma_cmd.h" +#include "udma_jfr.h" +#include "udma_jfs.h" +#include "udma_jfc.h" +#include "udma_jetty.h" + +bool well_known_jetty_pgsz_check = true; + +const char *state_name[] = { + "RESET", + "READY", + "SUSPENDED", + "ERROR", + "INVALID" +}; + +const char *to_state_name(enum ubcore_jetty_state state) +{ + if ((int)state >= (int)STATE_NUM) + return state_name[STATE_NUM]; + + return state_name[state]; +} + +static int udma_get_user_jetty_cmd(struct udma_dev *dev, struct udma_jetty *jetty, + struct ubcore_udata *udata, + struct udma_create_jetty_ucmd *ucmd) +{ + struct udma_context *uctx; + unsigned long byte; + + if (!udata) { + jetty->sq.jetty_type = (enum udma_jetty_type)UDMA_URMA_NORMAL_JETTY_TYPE; + return 0; + } + + if (!udata->udrv_data) { + dev_err(dev->dev, "jetty udata udrv_data is null.\n"); + return -EINVAL; + } + + if (!udata->udrv_data->in_addr || udata->udrv_data->in_len < sizeof(*ucmd)) { + dev_err(dev->dev, "jetty in_len (%u) or addr is invalid.\n", + udata->udrv_data->in_len); + return -EINVAL; + } + + byte = copy_from_user(ucmd, (void *)(uintptr_t)udata->udrv_data->in_addr, + sizeof(*ucmd)); + if (byte) { + dev_err(dev->dev, + "failed to copy jetty udata, byte = %lu.\n", byte); + return -EFAULT; + } + + uctx = to_udma_context(udata->uctx); + jetty->sq.udma_ctx = uctx; + jetty->sq.tid = uctx->tid; + jetty->jetty_addr = ucmd->jetty_addr; + jetty->pi_type = ucmd->pi_type; + jetty->sq.jetty_type = (enum udma_jetty_type)ucmd->jetty_type; + jetty->sq.non_pin = ucmd->non_pin; + + return 0; +} + +static int udma_get_jetty_buf(struct udma_dev *dev, struct udma_jetty *jetty, + struct ubcore_udata *udata, + struct ubcore_jetty_cfg *cfg, + struct udma_create_jetty_ucmd *ucmd) +{ + struct ubcore_jfs_cfg jfs_cfg = { + .depth = cfg->jfs_depth, + .trans_mode = cfg->trans_mode, + .priority = cfg->priority, + .max_sge = cfg->max_send_sge, + .max_rsge = cfg->max_send_rsge, + .max_inline_data = cfg->max_inline_data, + .rnr_retry = cfg->rnr_retry, + .err_timeout = cfg->err_timeout, + .jfs_context = cfg->jetty_context, + .jfc = cfg->send_jfc, + }; + int ret; + + jfs_cfg.flag.bs.lock_free = cfg->flag.bs.lock_free; + if (!udata) + jetty->jetty_addr = (uintptr_t)&jetty->sq; + + jetty->jfr = to_udma_jfr(cfg->jfr); + + ret = udata ? udma_alloc_u_sq_buf(dev, &jetty->sq, ucmd) : + udma_alloc_k_sq_buf(dev, &jetty->sq, &jfs_cfg); + if (ret) { + dev_err(dev->dev, "failed to get sq buf, ret = %d.\n", ret); + return ret; + } + jetty->sq.trans_mode = jfs_cfg.trans_mode; + jetty->sq.is_jetty = true; + + return ret; +} + +static void udma_init_jettyc(struct udma_dev *dev, struct ubcore_jetty_cfg *cfg, + struct udma_jetty *jetty, void *mb_buf) +{ + struct udma_jetty_ctx *ctx = (struct udma_jetty_ctx *)mb_buf; + struct udma_jfc *receive_jfc = to_udma_jfc(cfg->recv_jfc); + uint8_t i; + + ctx->state = JETTY_READY; + ctx->jfs_mode = JETTY; + ctx->type = to_udma_type(cfg->trans_mode); + ctx->sl = dev->udma_sl[UDMA_DEFAULT_SL_NUM]; + if (ctx->type == JETTY_RM || ctx->type == JETTY_RC) { + for (i = 0; i < dev->udma_total_sl_num; i++) { + if (cfg->priority == dev->udma_sl[i]) { + ctx->sl = cfg->priority; + break; + } + } + } else if (ctx->type == JETTY_UM) { + ctx->sl = dev->unic_sl[UDMA_DEFAULT_SL_NUM]; + for (i = 0; i < dev->unic_sl_num; i++) { + if (cfg->priority == dev->unic_sl[i]) { + ctx->sl = cfg->priority; + break; + } + } + } + ctx->sqe_base_addr_l = (jetty->sq.buf.addr >> SQE_VA_L_OFFSET) & + (uint32_t)SQE_VA_L_VALID_BIT; + ctx->sqe_base_addr_h = (jetty->sq.buf.addr >> SQE_VA_H_OFFSET) & + (uint32_t)SQE_VA_H_VALID_BIT; + ctx->sqe_token_id_l = jetty->sq.tid & (uint32_t)SQE_TOKEN_ID_L_MASK; + ctx->sqe_token_id_h = (jetty->sq.tid >> SQE_TOKEN_ID_H_OFFSET) & + (uint32_t)SQE_TOKEN_ID_H_MASK; + ctx->sqe_bb_shift = ilog2(roundup_pow_of_two(jetty->sq.buf.entry_cnt)); + ctx->tx_jfcn = cfg->send_jfc->id; + ctx->ta_timeout = to_ta_timeout(cfg->err_timeout); + + if (!!(dev->caps.feature & UDMA_CAP_FEATURE_RNR_RETRY)) + ctx->rnr_retry_num = cfg->rnr_retry; + + ctx->jfrn_l = jetty->jfr->rq.id; + ctx->jfrn_h = jetty->jfr->rq.id >> JETTY_CTX_JFRN_H_OFFSET; + ctx->rx_jfcn = cfg->recv_jfc->id; + ctx->user_data_l = jetty->jetty_addr; + ctx->user_data_h = jetty->jetty_addr >> UDMA_USER_DATA_H_OFFSET; + ctx->seid_idx = cfg->eid_index; + ctx->pi_type = jetty->pi_type ? 1 : 0; + + if (!!(dev->caps.feature & UDMA_CAP_FEATURE_JFC_INLINE)) + ctx->cqe_ie = receive_jfc->inline_en; + + ctx->err_mode = cfg->flag.bs.error_suspend; + ctx->cmp_odr = cfg->flag.bs.outorder_comp; + ctx->avail_sgmt_ost = AVAIL_SGMT_OST_INIT; + ctx->sqe_pld_tokenid = jetty->sq.tid & (uint32_t)SQE_PLD_TOKEN_ID_MASK; + ctx->next_send_ssn = get_random_u16(); + ctx->next_rcv_ssn = ctx->next_send_ssn; +} + +static int update_jetty_grp_ctx_valid(struct udma_dev *udma_dev, + struct udma_jetty_grp *jetty_grp) +{ + struct udma_jetty_grp_ctx ctx[UDMA_CTX_NUM]; + struct ubase_mbx_attr mbox_attr = {}; + int ret; + + ctx[0].valid = jetty_grp->valid; + /* jetty number indicates the location of the jetty with the largest ID. */ + ctx[0].jetty_number = fls(jetty_grp->valid) - 1; + memset(ctx + 1, 0xff, sizeof(ctx[1])); + ctx[1].valid = 0; + ctx[1].jetty_number = 0; + + mbox_attr.tag = jetty_grp->jetty_grp_id; + mbox_attr.op = UDMA_CMD_MODIFY_JETTY_GROUP_CONTEXT; + ret = post_mailbox_update_ctx(udma_dev, ctx, sizeof(ctx), &mbox_attr); + if (ret) + dev_err(udma_dev->dev, + "post mailbox update jetty grp ctx failed, ret = %d.\n", + ret); + + return ret; +} + +static uint32_t udma_get_jetty_grp_jetty_id(uint32_t *valid, uint32_t *next) +{ + uint32_t bit_idx; + + bit_idx = find_next_zero_bit((unsigned long *)valid, UDMA_BITS_PER_INT, *next); + if (bit_idx >= UDMA_BITS_PER_INT) + bit_idx = find_next_zero_bit((unsigned long *)valid, UDMA_BITS_PER_INT, 0); + + *next = (*next + 1) >= UDMA_BITS_PER_INT ? 0 : *next + 1; + + return bit_idx; +} + +static int add_jetty_to_grp(struct udma_dev *udma_dev, struct ubcore_jetty_group *jetty_grp, + struct udma_jetty_queue *sq, uint32_t cfg_id) +{ + struct udma_jetty_grp *udma_jetty_grp = to_udma_jetty_grp(jetty_grp); + uint32_t bit_idx = cfg_id - udma_jetty_grp->start_jetty_id; + int ret = 0; + + mutex_lock(&udma_jetty_grp->valid_lock); + + if (cfg_id == 0) + bit_idx = udma_get_jetty_grp_jetty_id(&udma_jetty_grp->valid, + &udma_jetty_grp->next_jetty_id); + + if (bit_idx >= UDMA_BITS_PER_INT || (udma_jetty_grp->valid & BIT(bit_idx))) { + dev_err(udma_dev->dev, + "jg(%u.%u) vallid %u is full or user id(%u) error", + udma_jetty_grp->jetty_grp_id, udma_jetty_grp->start_jetty_id, + udma_jetty_grp->valid, cfg_id); + ret = -ENOMEM; + goto out; + } + + udma_jetty_grp->valid |= BIT(bit_idx); + sq->id = udma_jetty_grp->start_jetty_id + bit_idx; + sq->jetty_grp = udma_jetty_grp; + + ret = update_jetty_grp_ctx_valid(udma_dev, udma_jetty_grp); + if (ret) { + dev_err(udma_dev->dev, + "update jetty grp ctx valid failed, jetty_grp id is %u.\n", + udma_jetty_grp->jetty_grp_id); + + udma_jetty_grp->valid &= ~BIT(bit_idx); + } +out: + mutex_unlock(&udma_jetty_grp->valid_lock); + + return ret; +} + +static void remove_jetty_from_grp(struct udma_dev *udma_dev, + struct udma_jetty *jetty) +{ + struct udma_jetty_grp *jetty_grp = jetty->sq.jetty_grp; + uint32_t bit_idx; + int ret; + + bit_idx = jetty->sq.id - jetty_grp->start_jetty_id; + if (bit_idx >= UDMA_BITS_PER_INT) { + dev_err(udma_dev->dev, + "jetty_id(%u) is not in jetty grp, start_jetty_id(%u).\n", + jetty->sq.id, jetty_grp->start_jetty_id); + return; + } + + mutex_lock(&jetty_grp->valid_lock); + jetty_grp->valid &= ~BIT(bit_idx); + jetty->sq.jetty_grp = NULL; + + ret = update_jetty_grp_ctx_valid(udma_dev, jetty_grp); + if (ret) + dev_err(udma_dev->dev, + "update jetty grp ctx valid failed, jetty_grp id is %u.\n", + jetty_grp->jetty_grp_id); + + mutex_unlock(&jetty_grp->valid_lock); +} + +static int udma_specify_rsvd_jetty_id(struct udma_dev *udma_dev, uint32_t cfg_id) +{ + struct udma_ida *ida_table = &udma_dev->rsvd_jetty_ida_table; + int id; + + id = ida_alloc_range(&ida_table->ida, cfg_id, cfg_id, GFP_KERNEL); + if (id < 0) { + dev_err(udma_dev->dev, "user specify id %u has been used, ret = %d.\n", cfg_id, id); + return id; + } + + return 0; +} + +static int udma_user_specify_jetty_id(struct udma_dev *udma_dev, uint32_t cfg_id) +{ + if (cfg_id < udma_dev->caps.jetty.start_idx) + return udma_specify_rsvd_jetty_id(udma_dev, cfg_id); + + return udma_specify_adv_id(udma_dev, &udma_dev->jetty_table.bitmap_table, + cfg_id); +} + +int udma_alloc_jetty_id(struct udma_dev *udma_dev, uint32_t *idx, + struct udma_res *jetty_res) +{ + struct udma_group_bitmap *bitmap = &udma_dev->jetty_table.bitmap_table; + struct ida *ida = &udma_dev->rsvd_jetty_ida_table.ida; + uint32_t min = jetty_res->start_idx; + uint32_t next = jetty_res->next_idx; + uint32_t max; + int ret; + + if (jetty_res->max_cnt == 0) { + dev_err(udma_dev->dev, "ida alloc failed max_cnt is 0.\n"); + return -EINVAL; + } + + max = jetty_res->start_idx + jetty_res->max_cnt - 1; + + if (jetty_res != &udma_dev->caps.jetty) { + ret = ida_alloc_range(ida, next, max, GFP_KERNEL); + if (ret < 0) { + ret = ida_alloc_range(ida, min, max, GFP_KERNEL); + if (ret < 0) { + dev_err(udma_dev->dev, + "ida alloc failed %d.\n", ret); + return ret; + } + } + + *idx = (uint32_t)ret; + } else { + ret = udma_adv_id_alloc(udma_dev, bitmap, idx, false, next); + if (ret) { + ret = udma_adv_id_alloc(udma_dev, bitmap, idx, false, min); + if (ret) { + dev_err(udma_dev->dev, + "bitmap alloc failed %d.\n", ret); + return ret; + } + } + } + + jetty_res->next_idx = (*idx + 1) > max ? min : (*idx + 1); + + return 0; +} + +static int udma_alloc_normal_jetty_id(struct udma_dev *udma_dev, uint32_t *idx) +{ + int ret; + + ret = udma_alloc_jetty_id(udma_dev, idx, &udma_dev->caps.jetty); + if (ret == 0) + return 0; + + ret = udma_alloc_jetty_id(udma_dev, idx, &udma_dev->caps.user_ctrl_normal_jetty); + if (ret == 0) + return 0; + + return udma_alloc_jetty_id(udma_dev, idx, &udma_dev->caps.public_jetty); +} + +#define CFGID_CHECK(a, b) ((a) >= (b).start_idx && (a) < (b).start_idx + (b).max_cnt) + +static int udma_verify_jetty_type_dwqe(struct udma_dev *udma_dev, + uint32_t cfg_id) +{ + if (!CFGID_CHECK(cfg_id, udma_dev->caps.stars_jetty)) { + dev_err(udma_dev->dev, + "user id %u error, cache lock st idx %u cnt %u.\n", + cfg_id, udma_dev->caps.stars_jetty.start_idx, + udma_dev->caps.stars_jetty.max_cnt); + return -EINVAL; + } + + return 0; +} + +static int udma_verify_jetty_type_ccu(struct udma_dev *udma_dev, + uint32_t cfg_id) +{ + if (!CFGID_CHECK(cfg_id, udma_dev->caps.ccu_jetty)) { + dev_err(udma_dev->dev, + "user id %u error, ccu st idx %u cnt %u.\n", + cfg_id, udma_dev->caps.ccu_jetty.start_idx, + udma_dev->caps.ccu_jetty.max_cnt); + return -EINVAL; + } + + return 0; +} + +static int udma_verify_jetty_type_normal(struct udma_dev *udma_dev, + uint32_t cfg_id) +{ + if (!CFGID_CHECK(cfg_id, udma_dev->caps.user_ctrl_normal_jetty)) { + dev_err(udma_dev->dev, + "user id %u error, user ctrl normal st idx %u cnt %u.\n", + cfg_id, + udma_dev->caps.user_ctrl_normal_jetty.start_idx, + udma_dev->caps.user_ctrl_normal_jetty.max_cnt); + return -EINVAL; + } + + return 0; +} + +static int udma_verify_jetty_type_urma_normal(struct udma_dev *udma_dev, + uint32_t cfg_id) +{ + if (!(CFGID_CHECK(cfg_id, udma_dev->caps.public_jetty) || + CFGID_CHECK(cfg_id, udma_dev->caps.hdc_jetty) || + CFGID_CHECK(cfg_id, udma_dev->caps.jetty))) { + dev_err(udma_dev->dev, + "user id %u error, ccu st idx %u cnt %u, stars st idx %u, normal st idx %u cnt %u.\n", + cfg_id, udma_dev->caps.ccu_jetty.start_idx, + udma_dev->caps.ccu_jetty.max_cnt, + udma_dev->caps.stars_jetty.start_idx, + udma_dev->caps.jetty.start_idx, + udma_dev->caps.jetty.max_cnt); + return -EINVAL; + } + + if (well_known_jetty_pgsz_check && PAGE_SIZE != UDMA_HW_PAGE_SIZE) { + dev_err(udma_dev->dev, "Does not support specifying Jetty ID on non-4KB page systems.\n"); + return -EINVAL; + } + + return 0; +} + +static int udma_verify_jetty_type(struct udma_dev *udma_dev, + enum udma_jetty_type jetty_type, uint32_t cfg_id) +{ + int (*udma_cfg_id_check[UDMA_JETTY_TYPE_MAX])(struct udma_dev *udma_dev, + uint32_t cfg_id) = { + udma_verify_jetty_type_dwqe, + udma_verify_jetty_type_ccu, + udma_verify_jetty_type_normal, + udma_verify_jetty_type_urma_normal + }; + + if (jetty_type < UDMA_JETTY_TYPE_MAX) { + if (!cfg_id) + return 0; + + return udma_cfg_id_check[jetty_type](udma_dev, cfg_id); + } + + dev_err(udma_dev->dev, "invalid jetty type 0x%x.\n", jetty_type); + return -EINVAL; +} + +static int udma_alloc_jetty_id_own(struct udma_dev *udma_dev, uint32_t *id, + enum udma_jetty_type jetty_type) +{ + int ret; + + switch (jetty_type) { + case UDMA_CACHE_LOCK_DWQE_JETTY_TYPE: + ret = udma_alloc_jetty_id(udma_dev, id, + &udma_dev->caps.stars_jetty); + break; + case UDMA_NORMAL_JETTY_TYPE: + ret = udma_alloc_jetty_id(udma_dev, id, + &udma_dev->caps.user_ctrl_normal_jetty); + break; + case UDMA_CCU_JETTY_TYPE: + ret = udma_alloc_jetty_id(udma_dev, id, &udma_dev->caps.ccu_jetty); + break; + default: + ret = udma_alloc_normal_jetty_id(udma_dev, id); + break; + } + + if (ret) + dev_err(udma_dev->dev, + "udma alloc jetty id own failed, type = %d, ret = %d.\n", + jetty_type, ret); + + return ret; +} + +int alloc_jetty_id(struct udma_dev *udma_dev, struct udma_jetty_queue *sq, + uint32_t cfg_id, struct ubcore_jetty_group *jetty_grp) +{ + int ret; + + if (udma_verify_jetty_type(udma_dev, sq->jetty_type, cfg_id)) + return -EINVAL; + + if (cfg_id > 0 && !jetty_grp) { + ret = udma_user_specify_jetty_id(udma_dev, cfg_id); + if (ret) + return ret; + + sq->id = cfg_id; + } else if (jetty_grp) { + ret = add_jetty_to_grp(udma_dev, jetty_grp, sq, cfg_id); + if (ret) { + dev_err(udma_dev->dev, + "add jetty to grp failed, ret = %d.\n", ret); + return ret; + } + } else { + ret = udma_alloc_jetty_id_own(udma_dev, &sq->id, sq->jetty_type); + } + + return ret; +} + +static void free_jetty_id(struct udma_dev *udma_dev, + struct udma_jetty *udma_jetty, bool is_grp) +{ + if (udma_jetty->sq.id < udma_dev->caps.jetty.start_idx) + udma_id_free(&udma_dev->rsvd_jetty_ida_table, udma_jetty->sq.id); + else if (is_grp) + remove_jetty_from_grp(udma_dev, udma_jetty); + else + udma_adv_id_free(&udma_dev->jetty_table.bitmap_table, + udma_jetty->sq.id, false); +} + +static void udma_dfx_store_jetty_id(struct udma_dev *udma_dev, + struct udma_jetty *udma_jetty) +{ + struct udma_dfx_jetty *jetty; + int ret; + + jetty = (struct udma_dfx_jetty *)xa_load(&udma_dev->dfx_info->jetty.table, + udma_jetty->sq.id); + if (jetty) { + dev_warn(udma_dev->dev, "jetty_id(%u) already exists in dfx.\n", + udma_jetty->sq.id); + return; + } + + jetty = kzalloc(sizeof(*jetty), GFP_KERNEL); + if (!jetty) + return; + + jetty->id = udma_jetty->sq.id; + jetty->jfs_depth = udma_jetty->sq.buf.entry_cnt / udma_jetty->sq.sqe_bb_cnt; + + write_lock(&udma_dev->dfx_info->jetty.rwlock); + ret = xa_err(xa_store(&udma_dev->dfx_info->jetty.table, udma_jetty->sq.id, + jetty, GFP_KERNEL)); + if (ret) { + write_unlock(&udma_dev->dfx_info->jetty.rwlock); + dev_err(udma_dev->dev, "store jetty_id(%u) to jetty_table failed in dfx.\n", + udma_jetty->sq.id); + kfree(jetty); + return; + } + + ++udma_dev->dfx_info->jetty.cnt; + write_unlock(&udma_dev->dfx_info->jetty.rwlock); +} + +static int +udma_alloc_jetty_sq(struct udma_dev *udma_dev, struct udma_jetty *jetty, + struct ubcore_jetty_cfg *cfg, struct ubcore_udata *udata) +{ + struct udma_create_jetty_ucmd ucmd = {}; + int ret; + + ret = udma_get_user_jetty_cmd(udma_dev, jetty, udata, &ucmd); + if (ret) { + dev_err(udma_dev->dev, + "udma get user jetty ucmd failed, ret = %d.\n", ret); + return ret; + } + + ret = alloc_jetty_id(udma_dev, &jetty->sq, cfg->id, cfg->jetty_grp); + if (ret) { + dev_err(udma_dev->dev, "alloc jetty id failed, ret = %d.\n", ret); + return ret; + } + jetty->ubcore_jetty.jetty_id.id = jetty->sq.id; + jetty->ubcore_jetty.jetty_cfg = *cfg; + + ret = udma_get_jetty_buf(udma_dev, jetty, udata, cfg, &ucmd); + if (ret) + free_jetty_id(udma_dev, jetty, !!cfg->jetty_grp); + + return ret; +} + +static void udma_free_jetty_id_buf(struct udma_dev *udma_dev, + struct udma_jetty *udma_jetty, + struct ubcore_jetty_cfg *cfg) +{ + udma_free_sq_buf(udma_dev, &udma_jetty->sq); + free_jetty_id(udma_dev, udma_jetty, !!cfg->jetty_grp); +} + +void udma_reset_sw_k_jetty_queue(struct udma_jetty_queue *sq) +{ + sq->kva_curr = sq->buf.kva; + sq->pi = 0; + sq->ci = 0; + sq->flush_flag = false; +} + +static int udma_create_hw_jetty_ctx(struct udma_dev *dev, struct udma_jetty *udma_jetty, + struct ubcore_jetty_cfg *cfg) +{ + struct ubase_mbx_attr attr = {}; + struct udma_jetty_ctx ctx = {}; + int ret; + + if (cfg->priority >= UDMA_MAX_PRIORITY) { + dev_err(dev->dev, "kernel mode jetty priority is out of range, priority is %u.\n", + cfg->priority); + return -EINVAL; + } + + udma_init_jettyc(dev, cfg, udma_jetty, &ctx); + + attr.tag = udma_jetty->sq.id; + attr.op = UDMA_CMD_CREATE_JFS_CONTEXT; + ret = post_mailbox_update_ctx(dev, &ctx, sizeof(ctx), &attr); + if (ret) + dev_err(dev->dev, + "post mailbox create jetty ctx failed, ret = %d.\n", ret); + + return ret; +} + +void udma_set_query_flush_time(struct udma_jetty_queue *sq, uint8_t err_timeout) +{ +#define UDMA_TA_TIMEOUT_MAX_INDEX 3 + uint32_t time[] = { + UDMA_TA_TIMEOUT_128MS, + UDMA_TA_TIMEOUT_1000MS, + UDMA_TA_TIMEOUT_8000MS, + UDMA_TA_TIMEOUT_64000MS, + }; + uint8_t index; + + index = to_ta_timeout(err_timeout); + if (index > UDMA_TA_TIMEOUT_MAX_INDEX) + index = UDMA_TA_TIMEOUT_MAX_INDEX; + + sq->ta_timeout = time[index]; +} + +struct ubcore_jetty *udma_create_jetty(struct ubcore_device *ub_dev, + struct ubcore_jetty_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(ub_dev); + struct udma_jetty *udma_jetty; + int ret; + + udma_jetty = kzalloc(sizeof(*udma_jetty), GFP_KERNEL); + if (!udma_jetty) + return NULL; + + ret = udma_alloc_jetty_sq(udma_dev, udma_jetty, cfg, udata); + if (ret) { + dev_err(udma_dev->dev, + "udma alloc jetty id buf failed, ret = %d.\n", ret); + goto err_alloc_jetty; + } + + ret = xa_err(xa_store(&udma_dev->jetty_table.xa, udma_jetty->sq.id, + &udma_jetty->sq, GFP_KERNEL)); + if (ret) { + dev_err(udma_dev->dev, + "store jetty sq(%u) to sq table failed, ret = %d.\n", + udma_jetty->sq.id, ret); + goto err_store_jetty_sq; + } + + ret = udma_create_hw_jetty_ctx(udma_dev, udma_jetty, cfg); + if (ret) { + dev_err(udma_dev->dev, + "post mailbox create jetty ctx failed, ret = %d.\n", ret); + goto err_create_hw_jetty; + } + + udma_set_query_flush_time(&udma_jetty->sq, cfg->err_timeout); + udma_jetty->sq.state = UBCORE_JETTY_STATE_READY; + refcount_set(&udma_jetty->ae_refcount, 1); + init_completion(&udma_jetty->ae_comp); + + if (dfx_switch) + udma_dfx_store_jetty_id(udma_dev, udma_jetty); + + return &udma_jetty->ubcore_jetty; +err_create_hw_jetty: + xa_erase(&udma_dev->jetty_table.xa, udma_jetty->sq.id); +err_store_jetty_sq: + udma_free_jetty_id_buf(udma_dev, udma_jetty, cfg); +err_alloc_jetty: + kfree(udma_jetty); + + return NULL; +} + +int udma_destroy_hw_jetty_ctx(struct udma_dev *dev, uint32_t jetty_id) +{ + struct ubase_mbx_attr attr = {}; + int ret; + + attr.tag = jetty_id; + attr.op = UDMA_CMD_DESTROY_JFS_CONTEXT; + ret = post_mailbox_update_ctx(dev, NULL, 0, &attr); + if (ret) + dev_err(dev->dev, + "post mailbox destroy jetty ctx failed, ret = %d.\n", ret); + + return ret; +} + +int udma_set_jetty_state(struct udma_dev *dev, uint32_t jetty_id, + enum jetty_state state) +{ + struct udma_jetty_ctx *ctx, *ctx_mask; + struct ubase_mbx_attr mbox_attr = {}; + struct ubase_cmd_mailbox *mailbox; + int ret; + + mailbox = udma_alloc_cmd_mailbox(dev); + if (!mailbox) { + dev_err(dev->dev, "failed to alloc mailbox for jettyc.\n"); + return -EINVAL; + } + + ctx = (struct udma_jetty_ctx *)mailbox->buf; + + /* Optimize chip access performance. */ + ctx_mask = (struct udma_jetty_ctx *)((char *)ctx + UDMA_JFS_MASK_OFFSET); + memset(ctx_mask, 0xff, sizeof(struct udma_jetty_ctx)); + ctx->state = state; + ctx_mask->state = 0; + + mbox_attr.tag = jetty_id; + mbox_attr.op = UDMA_CMD_MODIFY_JFS_CONTEXT; + ret = udma_post_mbox(dev, mailbox, &mbox_attr); + if (ret) + dev_err(dev->dev, + "failed to upgrade jettyc, ret = %d.\n", ret); + udma_free_cmd_mailbox(dev, mailbox); + + return ret; +} + +static int udma_query_jetty_ctx(struct udma_dev *dev, + struct udma_jetty_ctx *jfs_ctx, + uint32_t jetty_id) +{ + struct ubase_mbx_attr mbox_attr = {}; + struct ubase_cmd_mailbox *mailbox; + + mbox_attr.tag = jetty_id; + mbox_attr.op = UDMA_CMD_QUERY_JFS_CONTEXT; + mailbox = udma_mailbox_query_ctx(dev, &mbox_attr); + if (!mailbox) + return -ENOMEM; + memcpy((void *)jfs_ctx, mailbox->buf, sizeof(*jfs_ctx)); + + udma_free_cmd_mailbox(dev, mailbox); + + return 0; +} + +void udma_clean_cqe_for_jetty(struct udma_dev *dev, struct udma_jetty_queue *sq, + struct ubcore_jfc *send_jfc, + struct ubcore_jfc *recv_jfc) +{ + if (sq->buf.kva) { + if (send_jfc) + udma_clean_jfc(send_jfc, sq->id, dev); + + if (recv_jfc && recv_jfc != send_jfc) + udma_clean_jfc(recv_jfc, sq->id, dev); + } +} + +static bool udma_wait_timeout(uint32_t *sum_times, uint32_t times, uint32_t ta_timeout) +{ + uint32_t wait_time; + + if (*sum_times > ta_timeout) + return true; + + wait_time = 1 << times; + msleep(wait_time); + *sum_times += wait_time; + + return false; +} + +static void udma_mask_jetty_ctx(struct udma_jetty_ctx *ctx) +{ + ctx->sqe_base_addr_l = 0; + ctx->sqe_base_addr_h = 0; + ctx->user_data_l = 0; + ctx->user_data_h = 0; +} + +static bool udma_query_jetty_fd(struct udma_dev *dev, struct udma_jetty_queue *sq) +{ + struct udma_jetty_ctx ctx = {}; + uint16_t rcv_send_diff = 0; + uint32_t sum_times = 0; + uint32_t times = 0; + + while (true) { + if (udma_query_jetty_ctx(dev, &ctx, sq->id)) + return false; + + if (ctx.flush_cqe_done) + return true; + + if (udma_wait_timeout(&sum_times, times, UDMA_TA_TIMEOUT_64000MS)) + break; + + times++; + } + + /* In the flip scenario, ctx.next_rcv_ssn - ctx.next_send_ssn value is less than 512. */ + rcv_send_diff = ctx.next_rcv_ssn - ctx.next_send_ssn; + if (ctx.flush_ssn_vld && rcv_send_diff < UDMA_RCV_SEND_MAX_DIFF) + return true; + + udma_mask_jetty_ctx(&ctx); + udma_dfx_ctx_print(dev, "Flush Failed Jetty", sq->id, sizeof(ctx) / sizeof(uint32_t), + (uint32_t *)&ctx); + + return false; +} + +int udma_modify_jetty_precondition(struct udma_dev *dev, struct udma_jetty_queue *sq) +{ + struct udma_jetty_ctx ctx = {}; + uint16_t rcv_send_diff = 0; + uint32_t sum_times = 0; + uint32_t times = 0; + int ret; + + while (true) { + ret = udma_query_jetty_ctx(dev, &ctx, sq->id); + if (ret) { + dev_err(dev->dev, "query jetty ctx failed, id = %u, ret = %d.\n", + sq->id, ret); + return ret; + } + + rcv_send_diff = ctx.next_rcv_ssn - ctx.next_send_ssn; + if (ctx.PI == ctx.CI && rcv_send_diff < UDMA_RCV_SEND_MAX_DIFF && + ctx.state == JETTY_READY) + break; + + if (rcv_send_diff < UDMA_RCV_SEND_MAX_DIFF && + ctx.state == JETTY_ERROR) + break; + + if (udma_wait_timeout(&sum_times, times, sq->ta_timeout)) { + dev_warn(dev->dev, "TA timeout, id = %u. PI = %d, CI = %d, nxt_send_ssn = %d nxt_rcv_ssn = %d state = %d.\n", + sq->id, ctx.PI, ctx.CI, ctx.next_send_ssn, + ctx.next_rcv_ssn, ctx.state); + break; + } + times++; + } + + return 0; +} + +static bool udma_destroy_jetty_precondition(struct udma_dev *dev, struct udma_jetty_queue *sq) +{ +#define UDMA_DESTROY_JETTY_DELAY_TIME 100U + + if (sq->state != UBCORE_JETTY_STATE_READY && sq->state != UBCORE_JETTY_STATE_SUSPENDED) + goto query_jetty_fd; + + if (dev->caps.feature & UDMA_CAP_FEATURE_UE_RX_CLOSE) + goto modify_to_err; + + if (udma_modify_jetty_precondition(dev, sq)) + return false; + +modify_to_err: + if (udma_set_jetty_state(dev, sq->id, JETTY_ERROR)) { + dev_err(dev->dev, "modify jetty to error failed, id: %u.\n", + sq->id); + return false; + } + + sq->state = UBCORE_JETTY_STATE_ERROR; + +query_jetty_fd: + if (!udma_query_jetty_fd(dev, sq)) + return false; + + udelay(UDMA_DESTROY_JETTY_DELAY_TIME); + + return true; +} + +int udma_modify_and_destroy_jetty(struct udma_dev *dev, + struct udma_jetty_queue *sq) +{ + int ret; + + if (!udma_destroy_jetty_precondition(dev, sq)) + return -EFAULT; + + if (sq->state != UBCORE_JETTY_STATE_RESET) { + ret = udma_destroy_hw_jetty_ctx(dev, sq->id); + if (ret) { + dev_err(dev->dev, "jetty destroyed failed, id: %u.\n", + sq->id); + return ret; + } + } + + return 0; +} + +static void udma_free_jetty(struct ubcore_jetty *jetty) +{ + struct udma_dev *udma_dev = to_udma_dev(jetty->ub_dev); + struct udma_jetty *udma_jetty = to_udma_jetty(jetty); + + udma_clean_cqe_for_jetty(udma_dev, &udma_jetty->sq, jetty->jetty_cfg.send_jfc, + jetty->jetty_cfg.recv_jfc); + + if (dfx_switch) + udma_dfx_delete_id(udma_dev, &udma_dev->dfx_info->jetty, + udma_jetty->sq.id); + + xa_erase(&udma_dev->jetty_table.xa, udma_jetty->sq.id); + + if (refcount_dec_and_test(&udma_jetty->ae_refcount)) + complete(&udma_jetty->ae_comp); + wait_for_completion(&udma_jetty->ae_comp); + + udma_free_sq_buf(udma_dev, &udma_jetty->sq); + free_jetty_id(udma_dev, udma_jetty, !!udma_jetty->sq.jetty_grp); + kfree(udma_jetty); +} + +int udma_destroy_jetty(struct ubcore_jetty *jetty) +{ + struct udma_dev *udma_dev = to_udma_dev(jetty->ub_dev); + struct udma_jetty *udma_jetty = to_udma_jetty(jetty); + int ret; + + if (!udma_jetty->ue_rx_closed && udma_close_ue_rx(udma_dev, true, true, false, 0)) { + dev_err(udma_dev->dev, "close ue rx failed when destroying jetty.\n"); + return -EINVAL; + } + + ret = udma_modify_and_destroy_jetty(udma_dev, &udma_jetty->sq); + if (ret) { + dev_err(udma_dev->dev, "udma modify error and destroy jetty failed, id: %u.\n", + jetty->jetty_id.id); + if (!udma_jetty->ue_rx_closed) + udma_open_ue_rx(udma_dev, true, true, false, 0); + return ret; + } + + udma_free_jetty(jetty); + udma_open_ue_rx(udma_dev, true, true, false, 0); + + return 0; +} + +static int udma_batch_jetty_get_ack(struct udma_dev *dev, + struct udma_jetty_queue **sq_list, + uint32_t jetty_cnt, bool *jetty_flag, + int *bad_jetty_index) +{ + struct udma_jetty_ctx ctx = {}; + struct udma_jetty_queue *sq; + uint16_t rcv_send_diff = 0; + uint32_t i; + int ret; + + for (i = 0; i < jetty_cnt; i++) { + sq = sq_list[i]; + if (sq->state != UBCORE_JETTY_STATE_READY && + sq->state != UBCORE_JETTY_STATE_SUSPENDED) + continue; + + if (jetty_flag[i]) + continue; + + ret = udma_query_jetty_ctx(dev, &ctx, sq->id); + if (ret) { + dev_err(dev->dev, + "query jetty ctx failed, id = %u, ret = %d.\n", + sq->id, ret); + *bad_jetty_index = 0; + return ret; + } + + rcv_send_diff = ctx.next_rcv_ssn - ctx.next_send_ssn; + if (ctx.PI == ctx.CI && rcv_send_diff < UDMA_RCV_SEND_MAX_DIFF && + ctx.state == JETTY_READY) { + jetty_flag[i] = true; + continue; + } + + if (rcv_send_diff < UDMA_RCV_SEND_MAX_DIFF && + ctx.state == JETTY_ERROR) { + jetty_flag[i] = true; + continue; + } + + *bad_jetty_index = 0; + break; + } + + return (i == jetty_cnt) ? 0 : -EAGAIN; +} + +static uint32_t get_max_jetty_ta_timeout(struct udma_jetty_queue **sq_list, + uint32_t jetty_cnt) +{ + uint32_t max_timeout = 0; + uint32_t i; + + for (i = 0; i < jetty_cnt; i++) { + if (sq_list[i]->ta_timeout > max_timeout) + max_timeout = sq_list[i]->ta_timeout; + } + + return max_timeout; +} + +static bool udma_batch_query_jetty_fd(struct udma_dev *dev, + struct udma_jetty_queue **sq_list, + uint32_t jetty_cnt, int *bad_jetty_index) +{ + uint32_t ta_timeout = get_max_jetty_ta_timeout(sq_list, jetty_cnt); + struct udma_jetty_ctx ctx = {}; + struct udma_jetty_queue *sq; + uint16_t rcv_send_diff = 0; + uint32_t sum_times = 0; + uint32_t flush_cnt = 0; + bool all_query_done; + uint32_t times = 0; + bool *jetty_flag; + uint32_t i; + + jetty_flag = kcalloc(jetty_cnt, sizeof(bool), GFP_KERNEL); + if (!jetty_flag) { + *bad_jetty_index = 0; + return false; + } + + while (true) { + for (i = 0; i < jetty_cnt; i++) { + if (jetty_flag[i]) + continue; + + sq = sq_list[i]; + if (udma_query_jetty_ctx(dev, &ctx, sq->id)) { + kfree(jetty_flag); + *bad_jetty_index = 0; + return false; + } + + if (!ctx.flush_cqe_done) + continue; + + flush_cnt++; + jetty_flag[i] = true; + } + + if (flush_cnt == jetty_cnt) { + kfree(jetty_flag); + return true; + } + + if (udma_wait_timeout(&sum_times, times, ta_timeout)) + break; + + times++; + } + + all_query_done = true; + + for (i = 0; i < jetty_cnt; i++) { + if (jetty_flag[i]) + continue; + + sq = sq_list[i]; + if (udma_query_jetty_ctx(dev, &ctx, sq->id)) { + kfree(jetty_flag); + *bad_jetty_index = 0; + return false; + } + + rcv_send_diff = ctx.next_rcv_ssn - ctx.next_send_ssn; + if (ctx.flush_cqe_done || (ctx.flush_ssn_vld && + rcv_send_diff < UDMA_RCV_SEND_MAX_DIFF)) + continue; + + *bad_jetty_index = 0; + all_query_done = false; + + udma_mask_jetty_ctx(&ctx); + udma_dfx_ctx_print(dev, "Flush Failed Jetty", sq->id, + sizeof(ctx) / sizeof(uint32_t), (uint32_t *)&ctx); + break; + } + + kfree(jetty_flag); + + return all_query_done; +} + +static int batch_modify_jetty_to_error(struct udma_dev *dev, + struct udma_jetty_queue **sq_list, + uint32_t jetty_cnt, int *bad_jetty_index) +{ + struct udma_jetty_queue *sq; + uint32_t i; + int ret; + + for (i = 0; i < jetty_cnt; i++) { + sq = sq_list[i]; + if (sq->state == UBCORE_JETTY_STATE_ERROR || + sq->state == UBCORE_JETTY_STATE_RESET) + continue; + + ret = udma_set_jetty_state(dev, sq->id, JETTY_ERROR); + if (ret) { + dev_err(dev->dev, "modify jetty to error failed, id: %u.\n", + sq->id); + *bad_jetty_index = 0; + return ret; + } + + sq->state = UBCORE_JETTY_STATE_ERROR; + } + + return 0; +} + +static int udma_batch_modify_jetty_precondition(struct udma_dev *dev, + struct udma_jetty_queue **sq_list, + uint32_t jetty_cnt, int *bad_jetty_index) +{ + uint32_t ta_timeout = get_max_jetty_ta_timeout(sq_list, jetty_cnt); + uint32_t sum_times = 0; + uint32_t times = 0; + bool *jetty_flag; + int ret; + + jetty_flag = kcalloc(jetty_cnt, sizeof(bool), GFP_KERNEL); + if (!jetty_flag) { + *bad_jetty_index = 0; + return -ENOMEM; + } + + while (true) { + ret = udma_batch_jetty_get_ack(dev, sq_list, jetty_cnt, + jetty_flag, bad_jetty_index); + if (ret != -EAGAIN) { + kfree(jetty_flag); + return ret; + } + + if (udma_wait_timeout(&sum_times, times, ta_timeout)) { + dev_warn(dev->dev, + "timeout after %u ms, not all jetty get ack.\n", + sum_times); + break; + } + times++; + } + + kfree(jetty_flag); + + return 0; +} + +static bool udma_batch_destroy_jetty_precondition(struct udma_dev *dev, + struct udma_jetty_queue **sq_list, + uint32_t jetty_cnt, int *bad_jetty_index) +{ + if (!(dev->caps.feature & UDMA_CAP_FEATURE_UE_RX_CLOSE) && + udma_batch_modify_jetty_precondition(dev, sq_list, jetty_cnt, bad_jetty_index)) + return false; + + if (batch_modify_jetty_to_error(dev, sq_list, jetty_cnt, bad_jetty_index)) { + dev_err(dev->dev, "batch md jetty err failed.\n"); + return false; + } + + if (!udma_batch_query_jetty_fd(dev, sq_list, jetty_cnt, bad_jetty_index)) + return false; + + udelay(UDMA_DESTROY_JETTY_DELAY_TIME); + + return true; +} + +int udma_batch_modify_and_destroy_jetty(struct udma_dev *dev, + struct udma_jetty_queue **sq_list, + uint32_t jetty_cnt, int *bad_jetty_index) +{ + uint32_t i; + int ret; + + if (!udma_batch_destroy_jetty_precondition(dev, sq_list, jetty_cnt, bad_jetty_index)) + return -EFAULT; + + for (i = 0; i < jetty_cnt; i++) { + if (sq_list[i]->state != UBCORE_JETTY_STATE_RESET) { + ret = udma_destroy_hw_jetty_ctx(dev, sq_list[i]->id); + if (ret) { + dev_err(dev->dev, + "jetty destroyed failed, id: %u.\n", + sq_list[i]->id); + *bad_jetty_index = 0; + return ret; + } + + sq_list[i]->state = UBCORE_JETTY_STATE_RESET; + } + } + + return 0; +} + +int udma_destroy_jetty_batch(struct ubcore_jetty **jetty, int jetty_cnt, int *bad_jetty_index) +{ + struct udma_jetty_queue **sq_list; + struct udma_dev *udma_dev; + uint32_t i; + int ret; + + if (!jetty) { + pr_err("jetty array is null.\n"); + return -EINVAL; + } + + if (!jetty_cnt) { + pr_err("jetty cnt is 0.\n"); + return -EINVAL; + } + + udma_dev = to_udma_dev(jetty[0]->ub_dev); + + sq_list = kcalloc(1, sizeof(*sq_list) * jetty_cnt, GFP_KERNEL); + if (!sq_list) { + *bad_jetty_index = 0; + return -ENOMEM; + } + + for (i = 0; i < jetty_cnt; i++) + sq_list[i] = &(to_udma_jetty(jetty[i])->sq); + + ret = udma_batch_modify_and_destroy_jetty(udma_dev, sq_list, jetty_cnt, bad_jetty_index); + + kfree(sq_list); + + if (ret) { + dev_err(udma_dev->dev, + "udma batch modify error and destroy jetty failed.\n"); + return ret; + } + + for (i = 0; i < jetty_cnt; i++) + udma_free_jetty(jetty[i]); + + return 0; +} + +static int udma_check_jetty_grp_info(struct ubcore_tjetty_cfg *cfg, struct udma_dev *dev) +{ + if (cfg->type == UBCORE_JETTY_GROUP) { + if (cfg->trans_mode != UBCORE_TP_RM) { + dev_err(dev->dev, "import jg only support RM, transmode is %u.\n", + cfg->trans_mode); + return -EINVAL; + } + + if (cfg->policy != UBCORE_JETTY_GRP_POLICY_HASH_HINT) { + dev_err(dev->dev, "import jg only support hint, policy is %u.\n", + cfg->policy); + return -EINVAL; + } + } + + return 0; +} + +int udma_unimport_jetty(struct ubcore_tjetty *tjetty) +{ + struct udma_target_jetty *udma_tjetty = to_udma_tjetty(tjetty); + struct udma_dev *udma_dev = to_udma_dev(tjetty->ub_dev); + + if (!IS_ERR_OR_NULL(tjetty->vtpn)) { + dev_err(udma_dev->dev, + "the target jetty is still being used, id = %u.\n", + tjetty->cfg.id.id); + return -EINVAL; + } + + udma_tjetty->token_value = 0; + tjetty->cfg.token_value.token = 0; + kfree(udma_tjetty); + + return 0; +} + +bool verify_modify_jetty(enum ubcore_jetty_state jetty_state, + enum ubcore_jetty_state attr_state) +{ + switch (jetty_state) { + case UBCORE_JETTY_STATE_RESET: + return attr_state == UBCORE_JETTY_STATE_READY; + case UBCORE_JETTY_STATE_READY: + return attr_state == UBCORE_JETTY_STATE_ERROR || + attr_state == UBCORE_JETTY_STATE_SUSPENDED; + case UBCORE_JETTY_STATE_SUSPENDED: + return attr_state == UBCORE_JETTY_STATE_ERROR; + case UBCORE_JETTY_STATE_ERROR: + return attr_state == UBCORE_JETTY_STATE_RESET; + default: + break; + } + + return false; +} + +enum jetty_state to_jetty_state(enum ubcore_jetty_state state) +{ + switch (state) { + case UBCORE_JETTY_STATE_ERROR: + return JETTY_ERROR; + case UBCORE_JETTY_STATE_SUSPENDED: + return JETTY_SUSPEND; + default: + break; + } + + return STATE_NUM; +} + +static int udma_modify_jetty_state(struct udma_dev *udma_dev, struct udma_jetty *udma_jetty, + struct ubcore_jetty_attr *attr) +{ + int ret; + + switch (attr->state) { + case UBCORE_JETTY_STATE_RESET: + ret = udma_destroy_hw_jetty_ctx(udma_dev, udma_jetty->sq.id); + break; + case UBCORE_JETTY_STATE_READY: + ret = udma_create_hw_jetty_ctx(udma_dev, udma_jetty, + &udma_jetty->ubcore_jetty.jetty_cfg); + if (ret) + break; + + udma_reset_sw_k_jetty_queue(&udma_jetty->sq); + break; + default: + ret = udma_close_ue_rx(udma_dev, true, true, false, 0); + if (ret) + break; + + if (!(udma_dev->caps.feature & UDMA_CAP_FEATURE_UE_RX_CLOSE)) { + if (udma_modify_jetty_precondition(udma_dev, &udma_jetty->sq)) { + ret = -ENOMEM; + udma_open_ue_rx(udma_dev, true, true, false, 0); + break; + } + } + + ret = udma_set_jetty_state(udma_dev, udma_jetty->sq.id, + to_jetty_state(attr->state)); + if (ret) + udma_open_ue_rx(udma_dev, true, true, false, 0); + else + udma_jetty->ue_rx_closed = true; + break; + } + + return ret; +} + +int udma_modify_jetty(struct ubcore_jetty *jetty, struct ubcore_jetty_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(jetty->ub_dev); + struct udma_jetty *udma_jetty = to_udma_jetty(jetty); + int ret; + + if (!(attr->mask & UBCORE_JETTY_STATE)) { + dev_err(udma_dev->dev, "modify jetty mask is error or not set, jetty_id = %u.\n", + udma_jetty->sq.id); + return -EINVAL; + } + + if (udma_jetty->sq.state == attr->state) { + dev_info(udma_dev->dev, "jetty state has been %s.\n", to_state_name(attr->state)); + return 0; + } + + if (!verify_modify_jetty(udma_jetty->sq.state, attr->state)) { + dev_err(udma_dev->dev, "not support modify jetty state from %s to %s.\n", + to_state_name(udma_jetty->sq.state), to_state_name(attr->state)); + return -EINVAL; + } + + ret = udma_modify_jetty_state(udma_dev, udma_jetty, attr); + if (ret) { + dev_err(udma_dev->dev, "modify jetty %u state to %s failed.\n", + udma_jetty->sq.id, to_state_name(attr->state)); + return ret; + } + udma_jetty->sq.state = attr->state; + + return 0; +} + +static int udma_alloc_group_start_id(struct udma_dev *udma_dev, + struct udma_group_bitmap *bitmap_table, + uint32_t *start_jetty_id) +{ + int ret; + + ret = udma_adv_id_alloc(udma_dev, bitmap_table, start_jetty_id, true, + bitmap_table->grp_next); + if (ret) { + ret = udma_adv_id_alloc(udma_dev, bitmap_table, start_jetty_id, + true, bitmap_table->min); + if (ret) + return ret; + } + + bitmap_table->grp_next = (*start_jetty_id + NUM_JETTY_PER_GROUP) > + bitmap_table->max ? bitmap_table->min : + (*start_jetty_id + NUM_JETTY_PER_GROUP); + + return 0; +} + +static int udma_alloc_jetty_grp_id(struct udma_dev *udma_dev, + struct udma_jetty_grp *jetty_grp) +{ + int ret; + + ret = udma_alloc_group_start_id(udma_dev, &udma_dev->jetty_table.bitmap_table, + &jetty_grp->start_jetty_id); + if (ret) { + dev_err(udma_dev->dev, + "alloc jetty id for grp failed, ret = %d.\n", ret); + return ret; + } + + ret = udma_id_alloc_auto_grow(udma_dev, &udma_dev->jetty_grp_table.ida_table, + &jetty_grp->jetty_grp_id); + if (ret) { + dev_err(udma_dev->dev, + "alloc jetty grp id failed, ret = %d.\n", ret); + udma_adv_id_free(&udma_dev->jetty_table.bitmap_table, + jetty_grp->start_jetty_id, true); + return ret; + } + + jetty_grp->ubcore_jetty_grp.jetty_grp_id.id = jetty_grp->jetty_grp_id; + + return 0; +} + +struct ubcore_jetty_group *udma_create_jetty_grp(struct ubcore_device *dev, + struct ubcore_jetty_grp_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct ubase_mbx_attr mbox_attr = {}; + struct udma_jetty_grp_ctx ctx = {}; + struct udma_jetty_grp *jetty_grp; + int ret; + + if (cfg->policy != UBCORE_JETTY_GRP_POLICY_HASH_HINT) { + dev_err(udma_dev->dev, "policy %u not support.\n", cfg->policy); + return NULL; + } + + jetty_grp = kzalloc(sizeof(*jetty_grp), GFP_KERNEL); + if (!jetty_grp) + return NULL; + + ret = udma_alloc_jetty_grp_id(udma_dev, jetty_grp); + if (ret) + goto err_alloc_jetty_grp_id; + + ctx.start_jetty_id = jetty_grp->start_jetty_id; + + ret = xa_err(xa_store(&udma_dev->jetty_grp_table.xa, jetty_grp->jetty_grp_id, + jetty_grp, GFP_KERNEL)); + if (ret) { + dev_err(udma_dev->dev, "store jetty group(%u) failed, ret = %d.\n", + jetty_grp->jetty_grp_id, ret); + goto err_store_jetty_grp; + } + + mbox_attr.tag = jetty_grp->jetty_grp_id; + mbox_attr.op = UDMA_CMD_CREATE_JETTY_GROUP_CONTEXT; + ret = post_mailbox_update_ctx(udma_dev, &ctx, sizeof(ctx), &mbox_attr); + if (ret) { + dev_err(udma_dev->dev, + "post mailbox update jetty ctx failed, ret = %d.\n", ret); + goto err_post_mailbox; + } + + mutex_init(&jetty_grp->valid_lock); + refcount_set(&jetty_grp->ae_refcount, 1); + init_completion(&jetty_grp->ae_comp); + + if (dfx_switch) + udma_dfx_store_id(udma_dev, &udma_dev->dfx_info->jetty_grp, + jetty_grp->jetty_grp_id, "jetty_grp"); + + return &jetty_grp->ubcore_jetty_grp; +err_post_mailbox: + xa_erase(&udma_dev->jetty_grp_table.xa, jetty_grp->jetty_grp_id); +err_store_jetty_grp: + udma_id_free(&udma_dev->jetty_grp_table.ida_table, + jetty_grp->jetty_grp_id); +err_alloc_jetty_grp_id: + kfree(jetty_grp); + + return NULL; +} + +int udma_delete_jetty_grp(struct ubcore_jetty_group *jetty_grp) +{ + struct udma_jetty_grp *udma_jetty_grp = to_udma_jetty_grp(jetty_grp); + struct udma_dev *udma_dev = to_udma_dev(jetty_grp->ub_dev); + struct ubase_mbx_attr mbox_attr = {}; + int ret; + + mbox_attr.tag = udma_jetty_grp->jetty_grp_id; + mbox_attr.op = UDMA_CMD_DESTROY_JETTY_GROUP_CONTEXT; + ret = post_mailbox_update_ctx(udma_dev, NULL, 0, &mbox_attr); + if (ret) { + dev_err(udma_dev->dev, + "post mailbox destroy jetty group failed, ret = %d.\n", ret); + return ret; + } + + xa_erase(&udma_dev->jetty_grp_table.xa, udma_jetty_grp->jetty_grp_id); + + if (refcount_dec_and_test(&udma_jetty_grp->ae_refcount)) + complete(&udma_jetty_grp->ae_comp); + wait_for_completion(&udma_jetty_grp->ae_comp); + + if (dfx_switch) + udma_dfx_delete_id(udma_dev, &udma_dev->dfx_info->jetty_grp, + udma_jetty_grp->jetty_grp_id); + + if (udma_jetty_grp->valid != 0) + dev_err(udma_dev->dev, + "jetty group been used, jetty valid is 0x%x.\n", + udma_jetty_grp->valid); + + mutex_destroy(&udma_jetty_grp->valid_lock); + udma_id_free(&udma_dev->jetty_grp_table.ida_table, + udma_jetty_grp->jetty_grp_id); + udma_adv_id_free(&udma_dev->jetty_table.bitmap_table, + udma_jetty_grp->start_jetty_id, true); + kfree(udma_jetty_grp); + + return ret; +} + +int udma_flush_jetty(struct ubcore_jetty *jetty, int cr_cnt, struct ubcore_cr *cr) +{ + struct udma_dev *udma_dev = to_udma_dev(jetty->ub_dev); + struct udma_jetty *udma_jetty = to_udma_jetty(jetty); + struct udma_jetty_queue *sq = &udma_jetty->sq; + int n_flushed; + + if (!sq->flush_flag) + return 0; + + if (!sq->lock_free) + spin_lock(&sq->lock); + + for (n_flushed = 0; n_flushed < cr_cnt; n_flushed++) { + if (sq->ci == sq->pi) + break; + udma_flush_sq(udma_dev, sq, cr + n_flushed); + } + + if (!sq->lock_free) + spin_unlock(&sq->lock); + + return n_flushed; +} + +int udma_post_jetty_send_wr(struct ubcore_jetty *jetty, struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr) +{ + struct udma_dev *udma_dev = to_udma_dev(jetty->ub_dev); + struct udma_jetty *udma_jetty = to_udma_jetty(jetty); + int ret; + + ret = udma_post_sq_wr(udma_dev, &udma_jetty->sq, wr, bad_wr); + if (ret) + dev_err(udma_dev->dev, + "jetty post sq wr failed, ret = %d, jetty id = %u.\n", + ret, udma_jetty->sq.id); + + return ret; +} + +int udma_post_jetty_recv_wr(struct ubcore_jetty *jetty, struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr) +{ + struct udma_dev *udma_dev = to_udma_dev(jetty->ub_dev); + struct udma_jetty *udma_jetty = to_udma_jetty(jetty); + struct ubcore_jfr *jfr; + int ret; + + jfr = &udma_jetty->jfr->ubcore_jfr; + ret = udma_post_jfr_wr(jfr, wr, bad_wr); + if (ret) + dev_err(udma_dev->dev, + "jetty post jfr wr failed, ret = %d, jetty id = %u.\n", + ret, udma_jetty->sq.id); + + return ret; +} + +int udma_unbind_jetty(struct ubcore_jetty *jetty) +{ + struct udma_jetty *udma_jetty = to_udma_jetty(jetty); + + udma_jetty->sq.rc_tjetty = NULL; + + return 0; +} + +struct ubcore_tjetty *udma_import_jetty_ex(struct ubcore_device *ub_dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_active_tp_cfg *active_tp_cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(ub_dev); + struct udma_target_jetty *tjetty; + int ret = 0; + + if (cfg->type != UBCORE_JETTY_GROUP && cfg->type != UBCORE_JETTY) { + dev_err(udma_dev->dev, + "the jetty of the type %u cannot be imported in exp.\n", + cfg->type); + return NULL; + } + + ret = udma_check_jetty_grp_info(cfg, udma_dev); + if (ret) + return NULL; + + tjetty = kzalloc(sizeof(*tjetty), GFP_KERNEL); + if (!tjetty) + return NULL; + + if (cfg->flag.bs.token_policy != UBCORE_TOKEN_NONE) { + tjetty->token_value = cfg->token_value.token; + tjetty->token_value_valid = true; + } + + udma_swap_endian(cfg->id.eid.raw, tjetty->le_eid.raw, UBCORE_EID_SIZE); + + return &tjetty->ubcore_tjetty; +} + +int udma_bind_jetty_ex(struct ubcore_jetty *jetty, + struct ubcore_tjetty *tjetty, + struct ubcore_active_tp_cfg *active_tp_cfg, + struct ubcore_udata *udata) +{ + struct udma_jetty *udma_jetty = to_udma_jetty(jetty); + + udma_jetty->sq.rc_tjetty = tjetty; + + return 0; +} + +module_param(well_known_jetty_pgsz_check, bool, 0444); +MODULE_PARM_DESC(well_known_jetty_pgsz_check, + "Whether check the system page size. default: true(true:check; false: not check)"); diff --git a/drivers/ub/urma/hw/udma/udma_jetty.h b/drivers/ub/urma/hw/udma/udma_jetty.h new file mode 100644 index 0000000000000000000000000000000000000000..011711dc19263b0cff290251eaaf536663b8d6c8 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_jetty.h @@ -0,0 +1,284 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_JETTY_H__ +#define __UDMA_JETTY_H__ + +#include "udma_common.h" + +#define SQE_TOKEN_ID_L_MASK GENMASK(11, 0) +#define SQE_TOKEN_ID_H_OFFSET 12U +#define SQE_TOKEN_ID_H_MASK GENMASK(7, 0) +#define SQE_VA_L_OFFSET 12U +#define SQE_VA_L_VALID_BIT GENMASK(19, 0) +#define SQE_VA_H_OFFSET 32U +#define SQE_VA_H_VALID_BIT GENMASK(31, 0) +#define JETTY_CTX_JFRN_H_OFFSET 12 +#define AVAIL_SGMT_OST_INIT 512 +#define UDMA_JFS_MASK_OFFSET 128 + +#define SQE_PLD_TOKEN_ID_MASK GENMASK(19, 0) + +#define UDMA_TA_TIMEOUT_128MS 128 +#define UDMA_TA_TIMEOUT_1000MS 1000 +#define UDMA_TA_TIMEOUT_8000MS 8000 +#define UDMA_TA_TIMEOUT_64000MS 64000 + +#define UDMA_MAX_PRIORITY 16 + +enum jetty_state { + JETTY_RESET, + JETTY_READY, + JETTY_ERROR, + JETTY_SUSPEND, + STATE_NUM, +}; + +struct udma_jetty { + struct ubcore_jetty ubcore_jetty; + struct udma_jfr *jfr; + struct udma_jetty_queue sq; + uint64_t jetty_addr; + refcount_t ae_refcount; + struct completion ae_comp; + bool pi_type; + bool ue_rx_closed; +}; + +struct udma_target_jetty { + struct ubcore_tjetty ubcore_tjetty; + union ubcore_eid le_eid; + uint32_t token_value; + bool token_value_valid; +}; + +enum jfsc_mode { + JFS, + JETTY, +}; + +enum jetty_type { + JETTY_RAW_OR_NIC, + JETTY_UM, + JETTY_RC, + JETTY_RM, + JETTY_TYPE_RESERVED, +}; + +struct udma_jetty_ctx { + /* DW0 */ + uint32_t ta_timeout : 2; + uint32_t rnr_retry_num : 3; + uint32_t type : 3; + uint32_t sqe_bb_shift : 4; + uint32_t sl : 4; + uint32_t state : 3; + uint32_t jfs_mode : 1; + uint32_t sqe_token_id_l : 12; + /* DW1 */ + uint32_t sqe_token_id_h : 8; + uint32_t err_mode : 1; + uint32_t rsv : 1; + uint32_t cmp_odr : 1; + uint32_t rsv1 : 1; + uint32_t sqe_base_addr_l : 20; + /* DW2 */ + uint32_t sqe_base_addr_h; + /* DW3 */ + uint32_t rsv2; + /* DW4 */ + uint32_t tx_jfcn : 20; + uint32_t jfrn_l : 12; + /* DW5 */ + uint32_t jfrn_h : 8; + uint32_t rsv3 : 4; + uint32_t rx_jfcn : 20; + /* DW6 */ + uint32_t seid_idx : 10; + uint32_t pi_type : 1; + uint32_t rsv4 : 21; + /* DW7 */ + uint32_t user_data_l; + /* DW8 */ + uint32_t user_data_h; + /* DW9 */ + uint32_t sqe_position : 1; + uint32_t sqe_pld_position : 1; + uint32_t sqe_pld_tokenid : 20; + uint32_t rsv5 : 10; + /* DW10 */ + uint32_t tpn : 24; + uint32_t rsv6 : 8; + /* DW11 */ + uint32_t rmt_eid : 20; + uint32_t rsv7 : 12; + /* DW12 */ + uint32_t rmt_tokenid : 20; + uint32_t rsv8 : 12; + /* DW13 - DW15 */ + uint32_t rsv8_1[3]; + /* DW16 */ + uint32_t next_send_ssn : 16; + uint32_t src_order_wqe : 16; + /* DW17 */ + uint32_t src_order_ssn : 16; + uint32_t src_order_sgme_cnt : 16; + /* DW18 */ + uint32_t src_order_sgme_send_cnt : 16; + uint32_t CI : 16; + /* DW19 */ + uint32_t wqe_sgmt_send_cnt : 20; + uint32_t src_order_wqebb_num : 4; + uint32_t src_order_wqe_vld : 1; + uint32_t no_wqe_send_cnt : 4; + uint32_t so_lp_vld : 1; + uint32_t fence_lp_vld : 1; + uint32_t strong_fence_lp_vld : 1; + /* DW20 */ + uint32_t PI : 16; + uint32_t sq_db_doing : 1; + uint32_t ost_rce_credit : 15; + /* DW21 */ + uint32_t sq_db_retrying : 1; + uint32_t wmtp_rsv0 : 31; + /* DW22 */ + uint32_t wait_ack_timeout : 1; + uint32_t wait_rnr_timeout : 1; + uint32_t cqe_ie : 1; + uint32_t cqe_sz : 1; + uint32_t wml_rsv0 : 28; + /* DW23 */ + uint32_t wml_rsv1 : 32; + /* DW24 */ + uint32_t next_rcv_ssn : 16; + uint32_t next_cpl_bb_idx : 16; + /* DW25 */ + uint32_t next_cpl_sgmt_num : 20; + uint32_t we_rsv0 : 12; + /* DW26 */ + uint32_t next_cpl_bb_num : 4; + uint32_t next_cpl_cqe_en : 1; + uint32_t next_cpl_info_vld : 1; + uint32_t rpting_cqe : 1; + uint32_t not_rpt_cqe : 1; + uint32_t flush_ssn : 16; + uint32_t flush_ssn_vld : 1; + uint32_t flush_vld : 1; + uint32_t flush_cqe_done : 1; + uint32_t we_rsv1 : 5; + /* DW27 */ + uint32_t rcved_cont_ssn_num : 20; + uint32_t we_rsv2 : 12; + /* DW28 */ + uint32_t sq_timer; + /* DW29 */ + uint32_t rnr_cnt : 3; + uint32_t abt_ssn : 16; + uint32_t abt_ssn_vld : 1; + uint32_t taack_timeout_flag : 1; + uint32_t we_rsv3 : 9; + uint32_t err_type_l : 2; + /* DW30 */ + uint32_t err_type_h : 7; + uint32_t sq_flush_ssn : 16; + uint32_t we_rsv4 : 9; + /* DW31 */ + uint32_t avail_sgmt_ost : 10; + uint32_t read_op_cnt : 10; + uint32_t we_rsv5 : 12; + /* DW32 - DW63 */ + uint32_t taack_nack_bm[32]; +}; + +struct udma_jetty_grp_ctx { + uint32_t start_jetty_id : 16; + uint32_t rsv : 11; + uint32_t jetty_number : 5; + uint32_t valid; +}; + +static inline uint32_t to_udma_type(uint32_t trans_mode) +{ + switch (trans_mode) { + case UBCORE_TP_RM: + return JETTY_RM; + case UBCORE_TP_RC: + return JETTY_RC; + case UBCORE_TP_UM: + return JETTY_UM; + default: + return JETTY_TYPE_RESERVED; + } +} + +static inline struct udma_jetty *to_udma_jetty(struct ubcore_jetty *jetty) +{ + return container_of(jetty, struct udma_jetty, ubcore_jetty); +} + +static inline struct udma_jetty_grp *to_udma_jetty_grp(struct ubcore_jetty_group *jetty_grp) +{ + return container_of(jetty_grp, struct udma_jetty_grp, ubcore_jetty_grp); +} + +static inline struct udma_target_jetty *to_udma_tjetty(struct ubcore_tjetty *tjetty) +{ + return container_of(tjetty, struct udma_target_jetty, ubcore_tjetty); +} + +static inline struct udma_jetty *to_udma_jetty_from_queue(struct udma_jetty_queue *queue) +{ + return container_of(queue, struct udma_jetty, sq); +} + +enum jetty_state to_jetty_state(enum ubcore_jetty_state state); +const char *to_state_name(enum ubcore_jetty_state state); +bool verify_modify_jetty(enum ubcore_jetty_state jetty_state, + enum ubcore_jetty_state attr_state); +int alloc_jetty_id(struct udma_dev *udma_dev, struct udma_jetty_queue *sq, + uint32_t cfg_id, struct ubcore_jetty_group *jetty_grp); +struct ubcore_jetty *udma_create_jetty(struct ubcore_device *ub_dev, + struct ubcore_jetty_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jetty(struct ubcore_jetty *jetty); +int udma_destroy_jetty_batch(struct ubcore_jetty **jetty_arr, int jetty_num, int *bad_jetty_index); +int udma_unimport_jetty(struct ubcore_tjetty *tjetty); +int udma_modify_jetty(struct ubcore_jetty *jetty, struct ubcore_jetty_attr *attr, + struct ubcore_udata *udata); +struct ubcore_jetty_group *udma_create_jetty_grp(struct ubcore_device *dev, + struct ubcore_jetty_grp_cfg *cfg, + struct ubcore_udata *udata); +int udma_delete_jetty_grp(struct ubcore_jetty_group *jetty_grp); +int udma_flush_jetty(struct ubcore_jetty *jetty, int cr_cnt, struct ubcore_cr *cr); +int udma_set_jetty_state(struct udma_dev *dev, uint32_t jetty_id, + enum jetty_state state); +int udma_post_jetty_send_wr(struct ubcore_jetty *jetty, struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr); +int udma_post_jetty_recv_wr(struct ubcore_jetty *jetty, struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr); +int udma_unbind_jetty(struct ubcore_jetty *jetty); +void udma_reset_sw_k_jetty_queue(struct udma_jetty_queue *sq); +int udma_destroy_hw_jetty_ctx(struct udma_dev *dev, uint32_t jetty_id); +void udma_set_query_flush_time(struct udma_jetty_queue *sq, uint8_t err_timeout); +int udma_modify_and_destroy_jetty(struct udma_dev *dev, + struct udma_jetty_queue *sq); +int udma_alloc_jetty_id(struct udma_dev *udma_dev, uint32_t *idx, + struct udma_res *jetty_res); +int udma_modify_jetty_precondition(struct udma_dev *dev, struct udma_jetty_queue *sq); + +struct ubcore_tjetty *udma_import_jetty_ex(struct ubcore_device *ub_dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_active_tp_cfg *active_tp_cfg, + struct ubcore_udata *udata); +int udma_bind_jetty_ex(struct ubcore_jetty *jetty, + struct ubcore_tjetty *tjetty, + struct ubcore_active_tp_cfg *active_tp_cfg, + struct ubcore_udata *udata); +void udma_clean_cqe_for_jetty(struct udma_dev *dev, struct udma_jetty_queue *sq, + struct ubcore_jfc *send_jfc, + struct ubcore_jfc *recv_jfc); +int udma_batch_modify_and_destroy_jetty(struct udma_dev *dev, + struct udma_jetty_queue **sq_list, + uint32_t jetty_cnt, int *bad_jetty_index); + +#endif /* __UDMA_JETTY_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_jfc.c b/drivers/ub/urma/hw/udma/udma_jfc.c new file mode 100644 index 0000000000000000000000000000000000000000..50ef624629df90ed33cd13724591349b6f2de0ba --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_jfc.c @@ -0,0 +1,1117 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt + +#include +#include +#include +#include +#include +#include "udma_cmd.h" +#include "udma_common.h" +#include "udma_jetty.h" +#include "udma_jfr.h" +#include "udma_jfs.h" +#include "udma_ctx.h" +#include "udma_db.h" +#include +#include "udma_jfc.h" + +static void udma_construct_jfc_ctx(struct udma_dev *dev, + struct udma_jfc *jfc, + struct udma_jfc_ctx *ctx) +{ + memset(ctx, 0, sizeof(struct udma_jfc_ctx)); + + ctx->state = UDMA_JFC_STATE_VALID; + if (jfc_arm_mode) + ctx->arm_st = UDMA_CTX_NO_ARMED; + else + ctx->arm_st = UDMA_CTX_ALWAYS_ARMED; + ctx->shift = jfc->cq_shift - UDMA_JFC_DEPTH_SHIFT_BASE; + ctx->jfc_type = UDMA_NORMAL_JFC_TYPE; + if (!!(dev->caps.feature & UDMA_CAP_FEATURE_JFC_INLINE)) + ctx->inline_en = jfc->inline_en; + ctx->cqe_va_l = jfc->buf.addr >> CQE_VA_L_OFFSET; + ctx->cqe_va_h = jfc->buf.addr >> CQE_VA_H_OFFSET; + ctx->cqe_token_id = jfc->tid; + + if (cqe_mode) + ctx->cq_cnt_mode = UDMA_CQE_CNT_MODE_BY_CI_PI_GAP; + else + ctx->cq_cnt_mode = UDMA_CQE_CNT_MODE_BY_COUNT; + + ctx->ceqn = jfc->ceqn; + if (jfc->stars_en) { + ctx->stars_en = UDMA_STARS_SWITCH; + ctx->record_db_en = UDMA_NO_RECORD_EN; + } else { + ctx->record_db_en = UDMA_RECORD_EN; + ctx->record_db_addr_l = jfc->db.db_addr >> UDMA_DB_L_OFFSET; + ctx->record_db_addr_h = jfc->db.db_addr >> UDMA_DB_H_OFFSET; + } +} + +void udma_init_jfc_param(struct ubcore_jfc_cfg *cfg, + struct udma_jfc *jfc) +{ + jfc->base.id = jfc->jfcn; + jfc->base.jfc_cfg = *cfg; + jfc->ceqn = cfg->ceqn; + jfc->lock_free = cfg->flag.bs.lock_free; + jfc->inline_en = cfg->flag.bs.jfc_inline; + jfc->cq_shift = ilog2(jfc->buf.entry_cnt); +} + +int udma_check_jfc_cfg(struct udma_dev *dev, struct udma_jfc *jfc, + struct ubcore_jfc_cfg *cfg) +{ + if (!jfc->buf.entry_cnt || jfc->buf.entry_cnt > dev->caps.jfc.depth) { + dev_err(dev->dev, "invalid jfc depth = %u, cap depth = %u.\n", + jfc->buf.entry_cnt, dev->caps.jfc.depth); + return -EINVAL; + } + + if (jfc->buf.entry_cnt < UDMA_JFC_DEPTH_MIN) + jfc->buf.entry_cnt = UDMA_JFC_DEPTH_MIN; + + if (cfg->ceqn >= dev->caps.comp_vector_cnt) { + dev_err(dev->dev, "invalid ceqn = %u, cap ceq cnt = %u.\n", + cfg->ceqn, dev->caps.comp_vector_cnt); + return -EINVAL; + } + + return 0; +} + +static int udma_get_cmd_from_user(struct udma_create_jfc_ucmd *ucmd, + struct udma_dev *dev, + struct ubcore_udata *udata, + struct udma_jfc *jfc) +{ +#define UDMA_JFC_CQE_SHIFT 6 + unsigned long byte; + + if (!udata->udrv_data || !udata->udrv_data->in_addr) { + dev_err(dev->dev, "jfc udrv_data or in_addr is null.\n"); + return -EINVAL; + } + + byte = copy_from_user(ucmd, (void *)(uintptr_t)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (uint32_t)sizeof(*ucmd))); + if (byte) { + dev_err(dev->dev, + "failed to copy udata from user, byte = %lu.\n", byte); + return -EFAULT; + } + + jfc->mode = ucmd->mode; + jfc->ctx = to_udma_context(udata->uctx); + if (jfc->mode > UDMA_NORMAL_JFC_TYPE && jfc->mode < UDMA_KERNEL_STARS_JFC_TYPE) { + jfc->buf.entry_cnt = ucmd->buf_len; + return 0; + } + + jfc->db.db_addr = ucmd->db_addr; + jfc->buf.entry_cnt = ucmd->buf_len >> UDMA_JFC_CQE_SHIFT; + + return 0; +} + +static int udma_alloc_u_cq(struct udma_dev *dev, struct udma_create_jfc_ucmd *ucmd, + struct udma_jfc *jfc) +{ + int ret; + + if (ucmd->is_hugepage) { + jfc->buf.addr = ucmd->buf_addr; + if (udma_occupy_u_hugepage(jfc->ctx, (void *)jfc->buf.addr)) { + dev_err(dev->dev, "failed to create cq, va not map.\n"); + return -EINVAL; + } + jfc->buf.is_hugepage = true; + } else { + ret = pin_queue_addr(dev, ucmd->buf_addr, ucmd->buf_len, &jfc->buf); + if (ret) { + dev_err(dev->dev, "failed to pin queue for jfc, ret = %d.\n", ret); + return ret; + } + } + jfc->tid = jfc->ctx->tid; + + ret = udma_pin_sw_db(jfc->ctx, &jfc->db); + if (ret) { + dev_err(dev->dev, "failed to pin sw db for jfc, ret = %d.\n", ret); + goto err_pin_db; + } + + return 0; +err_pin_db: + if (ucmd->is_hugepage) + udma_return_u_hugepage(jfc->ctx, (void *)jfc->buf.addr); + else + unpin_queue_addr(jfc->buf.umem); + + return ret; +} + +static int udma_alloc_k_cq(struct udma_dev *dev, struct udma_jfc *jfc) +{ + int ret; + + if (!jfc->lock_free) + spin_lock_init(&jfc->lock); + + jfc->buf.entry_size = dev->caps.cqe_size; + jfc->tid = dev->tid; + ret = udma_k_alloc_buf(dev, &jfc->buf); + if (ret) { + dev_err(dev->dev, "failed to alloc cq buffer, id=%u.\n", jfc->jfcn); + return ret; + } + + ret = udma_alloc_sw_db(dev, &jfc->db, UDMA_JFC_TYPE_DB); + if (ret) { + dev_err(dev->dev, "failed to alloc sw db for jfc(%u).\n", jfc->jfcn); + udma_k_free_buf(dev, &jfc->buf); + } + + return ret; +} + +static void udma_free_cq(struct udma_dev *dev, struct udma_jfc *jfc) +{ + if (jfc->mode != UDMA_NORMAL_JFC_TYPE) { + udma_free_sw_db(dev, &jfc->db); + return; + } + + if (jfc->buf.kva) { + udma_k_free_buf(dev, &jfc->buf); + udma_free_sw_db(dev, &jfc->db); + } else { + if (jfc->buf.is_hugepage) + udma_return_u_hugepage(jfc->ctx, (void *)jfc->buf.addr); + else + unpin_queue_addr(jfc->buf.umem); + udma_unpin_sw_db(jfc->ctx, &jfc->db); + } +} + +int udma_post_create_jfc_mbox(struct udma_dev *dev, struct udma_jfc *jfc) +{ + struct ubase_mbx_attr mbox_attr = {}; + struct ubase_cmd_mailbox *mailbox; + int ret; + + mailbox = udma_alloc_cmd_mailbox(dev); + if (!mailbox) { + dev_err(dev->dev, "failed to alloc mailbox for JFCC.\n"); + return -ENOMEM; + } + + if (jfc->mode == UDMA_STARS_JFC_TYPE || jfc->mode == UDMA_CCU_JFC_TYPE || + jfc->mode == UDMA_KERNEL_STARS_JFC_TYPE) + jfc->stars_en = true; + udma_construct_jfc_ctx(dev, jfc, (struct udma_jfc_ctx *)mailbox->buf); + + mbox_attr.tag = jfc->jfcn; + mbox_attr.op = UDMA_CMD_CREATE_JFC_CONTEXT; + ret = udma_post_mbox(dev, mailbox, &mbox_attr); + if (ret) + dev_err(dev->dev, + "failed to post create JFC mailbox, ret = %d.\n", ret); + + udma_free_cmd_mailbox(dev, mailbox); + + return ret; +} + +static int udma_verify_stars_jfc_param(struct udma_dev *dev, + struct udma_ex_jfc_addr *jfc_addr, + struct udma_jfc *jfc) +{ + uint32_t size; + + if (!jfc_addr->cq_addr) { + dev_err(dev->dev, "CQE addr is wrong.\n"); + return -ENOMEM; + } + if (!jfc_addr->cq_len) { + dev_err(dev->dev, "CQE len is wrong.\n"); + return -EINVAL; + } + + size = jfc->buf.entry_cnt * dev->caps.cqe_size; + + if (size != jfc_addr->cq_len) { + dev_err(dev->dev, "cqe buff size is wrong, buf size = %u.\n", size); + return -EINVAL; + } + + return 0; +} + +static int udma_get_stars_jfc_buf(struct udma_dev *dev, struct udma_jfc *jfc) +{ + struct udma_ex_jfc_addr *jfc_addr = &dev->cq_addr_array[jfc->mode]; + int ret; + + jfc->tid = dev->tid; + + ret = udma_verify_stars_jfc_param(dev, jfc_addr, jfc); + if (ret) + return ret; + + jfc->buf.addr = (dma_addr_t)(uintptr_t)jfc_addr->cq_addr; + + ret = udma_alloc_sw_db(dev, &jfc->db, UDMA_JFC_TYPE_DB); + if (ret) { + dev_err(dev->dev, "failed to alloc sw db for jfc(%u).\n", jfc->jfcn); + return -ENOMEM; + } + + return ret; +} + +static int udma_create_stars_jfc(struct udma_dev *dev, + struct udma_jfc *jfc, + struct ubcore_jfc_cfg *cfg, + struct ubcore_udata *udata, + struct udma_create_jfc_ucmd *ucmd) +{ + unsigned long flags_store; + unsigned long flags_erase; + int ret; + + ret = udma_id_alloc_auto_grow(dev, &dev->jfc_table.ida_table, &jfc->jfcn); + if (ret) { + dev_err(dev->dev, "failed to alloc id for stars JFC.\n"); + return -ENOMEM; + } + + udma_init_jfc_param(cfg, jfc); + xa_lock_irqsave(&dev->jfc_table.xa, flags_store); + ret = xa_err(__xa_store(&dev->jfc_table.xa, jfc->jfcn, jfc, GFP_ATOMIC)); + xa_unlock_irqrestore(&dev->jfc_table.xa, flags_store); + if (ret) { + dev_err(dev->dev, + "failed to stored stars jfc id to jfc_table, jfcn: %u.\n", + jfc->jfcn); + goto err_store_jfcn; + } + + ret = udma_get_stars_jfc_buf(dev, jfc); + if (ret) + goto err_alloc_cqc; + + ret = udma_post_create_jfc_mbox(dev, jfc); + if (ret) + goto err_get_jfc_buf; + + refcount_set(&jfc->event_refcount, 1); + init_completion(&jfc->event_comp); + + if (dfx_switch) + udma_dfx_store_id(dev, &dev->dfx_info->jfc, jfc->jfcn, "jfc"); + + return 0; + +err_get_jfc_buf: + udma_free_sw_db(dev, &jfc->db); +err_alloc_cqc: + xa_lock_irqsave(&dev->jfc_table.xa, flags_erase); + __xa_erase(&dev->jfc_table.xa, jfc->jfcn); + xa_unlock_irqrestore(&dev->jfc_table.xa, flags_erase); +err_store_jfcn: + udma_id_free(&dev->jfc_table.ida_table, jfc->jfcn); + + return -ENOMEM; +} + +struct ubcore_jfc *udma_create_jfc(struct ubcore_device *ubcore_dev, + struct ubcore_jfc_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *dev = to_udma_dev(ubcore_dev); + struct udma_create_jfc_ucmd ucmd = {}; + unsigned long flags_store; + unsigned long flags_erase; + struct udma_jfc *jfc; + int ret; + + jfc = kzalloc(sizeof(struct udma_jfc), GFP_KERNEL); + if (!jfc) + return NULL; + + if (udata) { + ret = udma_get_cmd_from_user(&ucmd, dev, udata, jfc); + if (ret) + goto err_get_cmd; + } else { + jfc->arm_sn = 1; + jfc->buf.entry_cnt = cfg->depth ? roundup_pow_of_two(cfg->depth) : cfg->depth; + } + + ret = udma_check_jfc_cfg(dev, jfc, cfg); + if (ret) + goto err_get_cmd; + + if (jfc->mode == UDMA_STARS_JFC_TYPE || jfc->mode == UDMA_CCU_JFC_TYPE) { + if (udma_create_stars_jfc(dev, jfc, cfg, udata, &ucmd)) + goto err_get_cmd; + return &jfc->base; + } + + ret = udma_id_alloc_auto_grow(dev, &dev->jfc_table.ida_table, + &jfc->jfcn); + if (ret) + goto err_get_cmd; + + udma_init_jfc_param(cfg, jfc); + + xa_lock_irqsave(&dev->jfc_table.xa, flags_store); + ret = xa_err(__xa_store(&dev->jfc_table.xa, jfc->jfcn, jfc, GFP_ATOMIC)); + xa_unlock_irqrestore(&dev->jfc_table.xa, flags_store); + if (ret) { + dev_err(dev->dev, + "failed to stored jfc id to jfc_table, jfcn: %u.\n", + jfc->jfcn); + goto err_store_jfcn; + } + + ret = udata ? udma_alloc_u_cq(dev, &ucmd, jfc) : udma_alloc_k_cq(dev, jfc); + if (ret) + goto err_get_jfc_buf; + + ret = udma_post_create_jfc_mbox(dev, jfc); + if (ret) + goto err_alloc_cqc; + + refcount_set(&jfc->event_refcount, 1); + init_completion(&jfc->event_comp); + + if (dfx_switch) + udma_dfx_store_id(dev, &dev->dfx_info->jfc, jfc->jfcn, "jfc"); + + return &jfc->base; + +err_alloc_cqc: + jfc->base.uctx = (udata == NULL ? NULL : udata->uctx); + udma_free_cq(dev, jfc); +err_get_jfc_buf: + xa_lock_irqsave(&dev->jfc_table.xa, flags_erase); + __xa_erase(&dev->jfc_table.xa, jfc->jfcn); + xa_unlock_irqrestore(&dev->jfc_table.xa, flags_erase); +err_store_jfcn: + udma_id_free(&dev->jfc_table.ida_table, jfc->jfcn); +err_get_cmd: + kfree(jfc); + return NULL; +} + +static int udma_post_destroy_jfc_mbox(struct udma_dev *dev, uint32_t jfcn) +{ + struct ubase_mbx_attr mbox_attr = {}; + struct ubase_cmd_mailbox *mailbox; + struct udma_jfc_ctx *ctx; + int ret; + + mailbox = udma_alloc_cmd_mailbox(dev); + if (!mailbox) { + dev_err(dev->dev, "failed to alloc mailbox for JFCC.\n"); + return -ENOMEM; + } + + ctx = (struct udma_jfc_ctx *)mailbox->buf; + + mbox_attr.tag = jfcn; + mbox_attr.op = UDMA_CMD_DESTROY_JFC_CONTEXT; + ret = udma_post_mbox(dev, mailbox, &mbox_attr); + if (ret) + dev_err(dev->dev, + "failed to post destroy JFC mailbox, ret = %d.\n", + ret); + + udma_free_cmd_mailbox(dev, mailbox); + + return ret; +} + +static int udma_query_jfc_destroy_done(struct udma_dev *dev, uint32_t jfcn) +{ + struct ubase_mbx_attr mbox_attr = {}; + struct ubase_cmd_mailbox *mailbox; + struct udma_jfc_ctx *jfc_ctx; + int ret; + + mbox_attr.tag = jfcn; + mbox_attr.op = UDMA_CMD_QUERY_JFC_CONTEXT; + mailbox = udma_mailbox_query_ctx(dev, &mbox_attr); + if (!mailbox) + return -ENOMEM; + + jfc_ctx = (struct udma_jfc_ctx *)mailbox->buf; + ret = jfc_ctx->pi == jfc_ctx->wr_cqe_idx ? 0 : -EAGAIN; + + jfc_ctx->cqe_token_value = 0; + jfc_ctx->remote_token_value = 0; + udma_free_cmd_mailbox(dev, mailbox); + + return ret; +} + +static int udma_destroy_and_flush_jfc(struct udma_dev *dev, uint32_t jfcn) +{ +#define QUERY_MAX_TIMES 5 + uint32_t wait_times = 0; + int ret; + + ret = udma_post_destroy_jfc_mbox(dev, jfcn); + if (ret) { + dev_err(dev->dev, "failed to post mbox to destroy jfc, id: %u.\n", jfcn); + return ret; + } + + while (true) { + if (udma_query_jfc_destroy_done(dev, jfcn) == 0) + return 0; + if (wait_times > QUERY_MAX_TIMES) + break; + msleep(1 << wait_times); + wait_times++; + } + dev_err(dev->dev, "jfc flush timed out, id: %u.\n", jfcn); + + return -EFAULT; +} + +int udma_destroy_jfc(struct ubcore_jfc *jfc) +{ + struct udma_dev *dev = to_udma_dev(jfc->ub_dev); + struct udma_jfc *ujfc = to_udma_jfc(jfc); + unsigned long flags; + int ret; + + ret = udma_destroy_and_flush_jfc(dev, ujfc->jfcn); + if (ret) + return ret; + + xa_lock_irqsave(&dev->jfc_table.xa, flags); + __xa_erase(&dev->jfc_table.xa, ujfc->jfcn); + xa_unlock_irqrestore(&dev->jfc_table.xa, flags); + + if (refcount_dec_and_test(&ujfc->event_refcount)) + complete(&ujfc->event_comp); + wait_for_completion(&ujfc->event_comp); + + if (dfx_switch) + udma_dfx_delete_id(dev, &dev->dfx_info->jfc, jfc->id); + + udma_free_cq(dev, ujfc); + udma_id_free(&dev->jfc_table.ida_table, ujfc->jfcn); + kfree(ujfc); + + return 0; +} + +int udma_jfc_completion(struct notifier_block *nb, unsigned long jfcn, + void *data) +{ + struct auxiliary_device *adev = (struct auxiliary_device *)data; + struct udma_dev *udma_dev = get_udma_dev(adev); + struct ubcore_jfc *ubcore_jfc; + struct udma_jfc *udma_jfc; + + xa_lock(&udma_dev->jfc_table.xa); + udma_jfc = (struct udma_jfc *)xa_load(&udma_dev->jfc_table.xa, jfcn); + if (!udma_jfc) { + xa_unlock(&udma_dev->jfc_table.xa); + dev_warn(udma_dev->dev, + "Completion event for bogus jfcn %lu.\n", jfcn); + return -EINVAL; + } + + ++udma_jfc->arm_sn; + + ubcore_jfc = &udma_jfc->base; + if (ubcore_jfc->jfce_handler) { + refcount_inc(&udma_jfc->event_refcount); + xa_unlock(&udma_dev->jfc_table.xa); + ubcore_jfc->jfce_handler(ubcore_jfc); + if (refcount_dec_and_test(&udma_jfc->event_refcount)) + complete(&udma_jfc->event_comp); + } else { + xa_unlock(&udma_dev->jfc_table.xa); + } + + return 0; +} + +static int udma_get_cqe_period(uint16_t cqe_period) +{ + uint16_t period[] = { + UDMA_CQE_PERIOD_0, + UDMA_CQE_PERIOD_4, + UDMA_CQE_PERIOD_16, + UDMA_CQE_PERIOD_64, + UDMA_CQE_PERIOD_256, + UDMA_CQE_PERIOD_1024, + UDMA_CQE_PERIOD_4096, + UDMA_CQE_PERIOD_16384 + }; + uint32_t i; + + for (i = 0; i < ARRAY_SIZE(period); ++i) { + if (cqe_period == period[i]) + return i; + } + + return -EINVAL; +} + +static int udma_check_jfc_attr(struct udma_dev *udma_dev, struct ubcore_jfc_attr *attr) +{ + if (!(attr->mask & (UBCORE_JFC_MODERATE_COUNT | UBCORE_JFC_MODERATE_PERIOD))) { + dev_err(udma_dev->dev, + "udma modify jfc mask is not set or invalid.\n"); + return -EINVAL; + } + + if ((attr->mask & UBCORE_JFC_MODERATE_COUNT) && + (attr->moderate_count >= UDMA_CQE_COALESCE_CNT_MAX)) { + dev_err(udma_dev->dev, "udma cqe coalesce cnt %u is invalid.\n", + attr->moderate_count); + return -EINVAL; + } + + if ((attr->mask & UBCORE_JFC_MODERATE_PERIOD) && + (udma_get_cqe_period(attr->moderate_period) == -EINVAL)) { + dev_err(udma_dev->dev, "udma cqe coalesce period %u is invalid.\n", + attr->moderate_period); + return -EINVAL; + } + + return 0; +} + +static int udma_modify_jfc_attr(struct udma_dev *dev, uint32_t jfcn, + struct ubcore_jfc_attr *attr) +{ + struct udma_jfc_ctx *jfc_context, *ctx_mask; + struct ubase_mbx_attr mbox_attr = {}; + struct ubase_cmd_mailbox *mailbox; + int ret; + + mailbox = udma_alloc_cmd_mailbox(dev); + if (!mailbox) { + dev_err(dev->dev, "failed to alloc mailbox for modify jfc.\n"); + return -ENOMEM; + } + + jfc_context = &((struct udma_jfc_ctx *)mailbox->buf)[0]; + ctx_mask = &((struct udma_jfc_ctx *)mailbox->buf)[1]; + memset(ctx_mask, 0xff, sizeof(struct udma_jfc_ctx)); + + if (attr->mask & UBCORE_JFC_MODERATE_COUNT) { + jfc_context->cqe_coalesce_cnt = attr->moderate_count; + ctx_mask->cqe_coalesce_cnt = 0; + } + + if (attr->mask & UBCORE_JFC_MODERATE_PERIOD) { + jfc_context->cqe_coalesce_period = + udma_get_cqe_period(attr->moderate_period); + ctx_mask->cqe_coalesce_period = 0; + } + + mbox_attr.tag = jfcn; + mbox_attr.op = UDMA_CMD_MODIFY_JFC_CONTEXT; + ret = udma_post_mbox(dev, mailbox, &mbox_attr); + if (ret) + dev_err(dev->dev, + "failed to send post mbox in modify JFCC, ret = %d.\n", + ret); + + udma_free_cmd_mailbox(dev, mailbox); + + return ret; +} + +int udma_modify_jfc(struct ubcore_jfc *ubcore_jfc, struct ubcore_jfc_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_device = to_udma_dev(ubcore_jfc->ub_dev); + struct udma_jfc *udma_jfc = to_udma_jfc(ubcore_jfc); + int ret; + + ret = udma_check_jfc_attr(udma_device, attr); + if (ret) + return ret; + + ret = udma_modify_jfc_attr(udma_device, udma_jfc->jfcn, attr); + if (ret) + dev_err(udma_device->dev, + "failed to modify JFC, jfcn = %u, ret = %d.\n", + udma_jfc->jfcn, ret); + + return ret; +} + +int udma_rearm_jfc(struct ubcore_jfc *jfc, bool solicited_only) +{ + struct udma_dev *dev = to_udma_dev(jfc->ub_dev); + struct udma_jfc *udma_jfc = to_udma_jfc(jfc); + struct udma_jfc_db db; + + db.ci = udma_jfc->ci & (uint32_t)UDMA_JFC_DB_CI_IDX_M; + db.notify = solicited_only; + db.arm_sn = udma_jfc->arm_sn; + db.type = UDMA_CQ_ARM_DB; + db.jfcn = udma_jfc->jfcn; + + udma_write64(dev, (uint64_t *)&db, (void __iomem *)(dev->k_db_base + + UDMA_JFC_HW_DB_OFFSET)); + + return 0; +} + +static enum jfc_poll_state udma_get_cr_status(struct udma_dev *dev, + uint8_t src_status, + uint8_t substatus, + enum ubcore_cr_status *dst_status) +{ +#define UDMA_SRC_STATUS_NUM 7 +#define UDMA_SUB_STATUS_NUM 5 + +struct udma_cr_status { + bool is_valid; + enum ubcore_cr_status cr_status; +}; + + static struct udma_cr_status map[UDMA_SRC_STATUS_NUM][UDMA_SUB_STATUS_NUM] = { + {{true, UBCORE_CR_SUCCESS}, {false, UBCORE_CR_SUCCESS}, + {false, UBCORE_CR_SUCCESS}, {false, UBCORE_CR_SUCCESS}, + {false, UBCORE_CR_SUCCESS}}, + {{true, UBCORE_CR_UNSUPPORTED_OPCODE_ERR}, {false, UBCORE_CR_SUCCESS}, + {false, UBCORE_CR_SUCCESS}, {false, UBCORE_CR_SUCCESS}, + {false, UBCORE_CR_SUCCESS}}, + {{false, UBCORE_CR_SUCCESS}, {true, UBCORE_CR_LOC_LEN_ERR}, + {true, UBCORE_CR_LOC_ACCESS_ERR}, {true, UBCORE_CR_REM_RESP_LEN_ERR}, + {true, UBCORE_CR_LOC_DATA_POISON}}, + {{false, UBCORE_CR_SUCCESS}, {true, UBCORE_CR_REM_UNSUPPORTED_REQ_ERR}, + {true, UBCORE_CR_REM_ACCESS_ABORT_ERR}, {false, UBCORE_CR_SUCCESS}, + {true, UBCORE_CR_REM_DATA_POISON}}, + {{true, UBCORE_CR_RNR_RETRY_CNT_EXC_ERR}, {false, UBCORE_CR_SUCCESS}, + {false, UBCORE_CR_SUCCESS}, {false, UBCORE_CR_SUCCESS}, + {false, UBCORE_CR_SUCCESS}}, + {{true, UBCORE_CR_ACK_TIMEOUT_ERR}, {false, UBCORE_CR_SUCCESS}, + {false, UBCORE_CR_SUCCESS}, {false, UBCORE_CR_SUCCESS}, + {false, UBCORE_CR_SUCCESS}}, + {{true, UBCORE_CR_FLUSH_ERR}, {false, UBCORE_CR_SUCCESS}, + {false, UBCORE_CR_SUCCESS}, {false, UBCORE_CR_SUCCESS}, + {false, UBCORE_CR_SUCCESS}} + }; + + if ((src_status < UDMA_SRC_STATUS_NUM) && (substatus < UDMA_SUB_STATUS_NUM) && + map[src_status][substatus].is_valid) { + *dst_status = map[src_status][substatus].cr_status; + return JFC_OK; + } + + dev_err(dev->dev, "cqe status is error, status = %u, substatus = %u.\n", + src_status, substatus); + + return JFC_POLL_ERR; +} + +static void udma_handle_inline_cqe(struct udma_jfc_cqe *cqe, uint8_t opcode, + struct udma_jetty_queue *queue, + struct ubcore_cr *cr) +{ + struct udma_jfr *jfr = to_udma_jfr_from_queue(queue); + uint32_t rqe_idx, data_len, sge_idx, size; + struct udma_wqe_sge *sge_list; + void *cqe_inline_buf; + + rqe_idx = cqe->entry_idx; + sge_list = (struct udma_wqe_sge *)(jfr->rq.buf.kva + + rqe_idx * jfr->rq.buf.entry_size); + data_len = cqe->byte_cnt; + cqe_inline_buf = opcode == HW_CQE_OPC_SEND ? + (void *)&cqe->data_l : (void *)&cqe->inline_data; + + for (sge_idx = 0; (sge_idx < jfr->max_sge) && data_len; sge_idx++) { + size = sge_list[sge_idx].length < data_len ? + sge_list[sge_idx].length : data_len; + memcpy((void *)(uintptr_t)sge_list[sge_idx].va, + cqe_inline_buf, size); + data_len -= size; + cqe_inline_buf += size; + } + cr->completion_len = cqe->byte_cnt - data_len; + + if (data_len) { + cqe->status = UDMA_CQE_LOCAL_OP_ERR; + cqe->substatus = UDMA_CQE_LOCAL_LENGTH_ERR; + } +} + +static void udma_parse_opcode_for_res(struct udma_dev *dev, + struct udma_jfc_cqe *cqe, + struct ubcore_cr *cr, + struct list_head *tid_list) +{ + uint8_t opcode = cqe->opcode; + struct udma_inv_tid *inv_tid; + + switch (opcode) { + case HW_CQE_OPC_SEND: + cr->opcode = UBCORE_CR_OPC_SEND; + break; + case HW_CQE_OPC_SEND_WITH_IMM: + cr->imm_data = (uint64_t)cqe->data_h << UDMA_IMM_DATA_SHIFT | + cqe->data_l; + cr->opcode = UBCORE_CR_OPC_SEND_WITH_IMM; + break; + case HW_CQE_OPC_SEND_WITH_INV: + cr->invalid_token.token_id = cqe->data_l & (uint32_t)UDMA_CQE_INV_TOKEN_ID; + cr->invalid_token.token_id <<= UDMA_TID_SHIFT; + cr->invalid_token.token_value.token = cqe->data_h; + cr->opcode = UBCORE_CR_OPC_SEND_WITH_INV; + + inv_tid = kzalloc(sizeof(*inv_tid), GFP_ATOMIC); + if (!inv_tid) + return; + + inv_tid->tid = cr->invalid_token.token_id >> UDMA_TID_SHIFT; + list_add(&inv_tid->list, tid_list); + + break; + case HW_CQE_OPC_WRITE_WITH_IMM: + cr->imm_data = (uint64_t)cqe->data_h << UDMA_IMM_DATA_SHIFT | + cqe->data_l; + cr->opcode = UBCORE_CR_OPC_WRITE_WITH_IMM; + break; + default: + cr->opcode = (enum ubcore_cr_opcode)HW_CQE_OPC_ERR; + dev_err(dev->dev, "receive invalid opcode :%u.\n", opcode); + cr->status = UBCORE_CR_UNSUPPORTED_OPCODE_ERR; + break; + } +} + +static struct udma_jfr *udma_get_jfr(struct udma_dev *udma_dev, + struct udma_jfc_cqe *cqe, + struct ubcore_cr *cr) +{ + struct udma_jetty_queue *udma_sq; + struct udma_jetty *jetty = NULL; + struct udma_jfr *jfr = NULL; + uint32_t local_id; + + local_id = cr->local_id; + if (cqe->is_jetty) { + udma_sq = (struct udma_jetty_queue *)xa_load(&udma_dev->jetty_table.xa, local_id); + if (!udma_sq) { + dev_warn(udma_dev->dev, + "get jetty failed, jetty_id = %u.\n", local_id); + return NULL; + } + jetty = to_udma_jetty_from_queue(udma_sq); + jfr = jetty->jfr; + cr->user_data = (uintptr_t)&jetty->ubcore_jetty; + } else { + jfr = (struct udma_jfr *)xa_load(&udma_dev->jfr_table.xa, local_id); + if (!jfr) { + dev_warn(udma_dev->dev, + "get jfr failed jfr id = %u.\n", local_id); + return NULL; + } + cr->user_data = (uintptr_t)&jfr->ubcore_jfr; + } + + return jfr; +} + +static bool udma_update_jfr_idx(struct udma_dev *dev, + struct udma_jfc_cqe *cqe, + struct ubcore_cr *cr, + bool is_clean) +{ + struct udma_jetty_queue *queue; + uint8_t opcode = cqe->opcode; + struct udma_jfr *jfr; + uint32_t entry_idx; + + jfr = udma_get_jfr(dev, cqe, cr); + if (!jfr) + return true; + + queue = &jfr->rq; + entry_idx = cqe->entry_idx; + cr->user_ctx = queue->wrid[entry_idx & (queue->buf.entry_cnt - (uint32_t)1)]; + + if (!is_clean && cqe->inline_en) + udma_handle_inline_cqe(cqe, opcode, queue, cr); + + if (!jfr->ubcore_jfr.jfr_cfg.flag.bs.lock_free) + spin_lock(&jfr->lock); + + udma_id_free(&jfr->idx_que.jfr_idx_table.ida_table, entry_idx); + queue->ci++; + + if (!jfr->ubcore_jfr.jfr_cfg.flag.bs.lock_free) + spin_unlock(&jfr->lock); + + return false; +} + +static enum jfc_poll_state udma_parse_cqe_for_send(struct udma_dev *dev, + struct udma_jfc_cqe *cqe, + struct ubcore_cr *cr) +{ + struct udma_jetty_queue *queue; + struct udma_jetty *jetty; + struct udma_jfs *jfs; + + queue = (struct udma_jetty_queue *)(uintptr_t)( + (uint64_t)cqe->user_data_h << UDMA_ADDR_SHIFT | + cqe->user_data_l); + if (!queue) { + dev_err(dev->dev, "jetty queue addr is null, jetty_id = %u.\n", cr->local_id); + return JFC_POLL_ERR; + } + + if (unlikely(udma_get_cr_status(dev, cqe->status, cqe->substatus, &cr->status))) + return JFC_POLL_ERR; + + if (!!cqe->fd) { + cr->status = UBCORE_CR_WR_FLUSH_ERR_DONE; + queue->flush_flag = true; + } else { + queue->ci += (cqe->entry_idx - queue->ci) & (queue->buf.entry_cnt - 1); + cr->user_ctx = queue->wrid[queue->ci & (queue->buf.entry_cnt - 1)]; + queue->ci++; + } + + if (!!cr->flag.bs.jetty) { + jetty = to_udma_jetty_from_queue(queue); + cr->user_data = (uintptr_t)&jetty->ubcore_jetty; + } else { + jfs = container_of(queue, struct udma_jfs, sq); + cr->user_data = (uintptr_t)&jfs->ubcore_jfs; + } + + return JFC_OK; +} + +static enum jfc_poll_state udma_parse_cqe_for_recv(struct udma_dev *dev, + struct udma_jfc_cqe *cqe, + struct ubcore_cr *cr, + struct list_head *tid_list) +{ + uint8_t substatus; + uint8_t status; + + if (unlikely(udma_update_jfr_idx(dev, cqe, cr, false))) + return JFC_POLL_ERR; + + udma_parse_opcode_for_res(dev, cqe, cr, tid_list); + status = cqe->status; + substatus = cqe->substatus; + if (unlikely(udma_get_cr_status(dev, status, substatus, &cr->status))) + return JFC_POLL_ERR; + + return JFC_OK; +} + +static enum jfc_poll_state parse_cqe_for_jfc(struct udma_dev *dev, + struct udma_jfc_cqe *cqe, + struct ubcore_cr *cr, + struct list_head *tid_list) +{ + enum jfc_poll_state ret; + + cr->flag.bs.s_r = cqe->s_r; + cr->flag.bs.jetty = cqe->is_jetty; + cr->completion_len = cqe->byte_cnt; + cr->tpn = cqe->tpn; + cr->local_id = cqe->local_num_h << UDMA_SRC_IDX_SHIFT | cqe->local_num_l; + cr->remote_id.id = cqe->rmt_idx; + udma_swap_endian((uint8_t *)(cqe->rmt_eid), cr->remote_id.eid.raw, UBCORE_EID_SIZE); + + if (cqe->s_r == CQE_FOR_RECEIVE) + ret = udma_parse_cqe_for_recv(dev, cqe, cr, tid_list); + else + ret = udma_parse_cqe_for_send(dev, cqe, cr); + + return ret; +} + +static struct udma_jfc_cqe *get_next_cqe(struct udma_jfc *jfc, uint32_t n) +{ + struct udma_jfc_cqe *cqe; + uint32_t valid_owner; + + cqe = (struct udma_jfc_cqe *)get_buf_entry(&jfc->buf, n); + + valid_owner = (n >> jfc->cq_shift) & UDMA_JFC_DB_VALID_OWNER_M; + if (!(cqe->owner ^ valid_owner)) + return NULL; + + return cqe; +} + +static void dump_cqe_aux_info(struct udma_dev *dev, struct ubcore_cr *cr) +{ + struct ubcore_user_ctl_out out = {}; + struct ubcore_user_ctl_in in = {}; + struct udma_cqe_info_in info_in; + + info_in.status = cr->status; + info_in.s_r = cr->flag.bs.s_r; + in.addr = (uint64_t)&info_in; + in.len = sizeof(struct udma_cqe_info_in); + in.opcode = UDMA_USER_CTL_QUERY_CQE_AUX_INFO; + + (void)udma_query_cqe_aux_info(&dev->ub_dev, NULL, &in, &out); +} + +static enum jfc_poll_state udma_poll_one(struct udma_dev *dev, + struct udma_jfc *jfc, + struct ubcore_cr *cr, + struct list_head *tid_list) +{ + struct udma_jfc_cqe *cqe; + + cqe = get_next_cqe(jfc, jfc->ci); + if (!cqe) + return JFC_EMPTY; + + ++jfc->ci; + /* Memory barrier */ + rmb(); + + if (parse_cqe_for_jfc(dev, cqe, cr, tid_list)) + return JFC_POLL_ERR; + + if (unlikely(cr->status != UBCORE_CR_SUCCESS) && dump_aux_info) + dump_cqe_aux_info(dev, cr); + + return JFC_OK; +} + +static void udma_inv_tid(struct udma_dev *dev, struct list_head *tid_list) +{ + struct udma_inv_tid *tid_node; + struct udma_inv_tid *tmp; + struct iommu_sva *ksva; + uint32_t tid; + + mutex_lock(&dev->ksva_mutex); + list_for_each_entry_safe(tid_node, tmp, tid_list, list) { + tid = tid_node->tid; + ksva = (struct iommu_sva *)xa_load(&dev->ksva_table, tid); + if (!ksva) { + dev_warn(dev->dev, "tid may have been released.\n"); + } else { + ummu_ksva_unbind_device(ksva); + __xa_erase(&dev->ksva_table, tid); + } + + list_del(&tid_node->list); + kfree(tid_node); + } + mutex_unlock(&dev->ksva_mutex); +} + +/* thanks to drivers/infiniband/hw/bnxt_re/ib_verbs.c */ +int udma_poll_jfc(struct ubcore_jfc *jfc, int cr_cnt, struct ubcore_cr *cr) +{ + struct udma_dev *dev = to_udma_dev(jfc->ub_dev); + struct udma_jfc *udma_jfc = to_udma_jfc(jfc); + enum jfc_poll_state err = JFC_OK; + struct list_head tid_list; + unsigned long flags; + uint32_t ci; + int npolled; + + INIT_LIST_HEAD(&tid_list); + + if (!jfc->jfc_cfg.flag.bs.lock_free) + spin_lock_irqsave(&udma_jfc->lock, flags); + + for (npolled = 0; npolled < cr_cnt; ++npolled) { + err = udma_poll_one(dev, udma_jfc, cr + npolled, &tid_list); + if (err != JFC_OK) + break; + } + + if (npolled) { + ci = udma_jfc->ci; + *udma_jfc->db.db_record = ci & (uint32_t)UDMA_JFC_DB_CI_IDX_M; + } + + if (!jfc->jfc_cfg.flag.bs.lock_free) + spin_unlock_irqrestore(&udma_jfc->lock, flags); + + if (!list_empty(&tid_list)) + udma_inv_tid(dev, &tid_list); + + return err == JFC_POLL_ERR ? -UDMA_INTER_ERR : npolled; +} + +void udma_clean_jfc(struct ubcore_jfc *jfc, uint32_t jetty_id, struct udma_dev *udma_dev) +{ + struct udma_jfc *udma_jfc = to_udma_jfc(jfc); + struct udma_jfc_cqe *dest; + struct udma_jfc_cqe *cqe; + struct ubcore_cr cr; + uint32_t nfreed = 0; + uint32_t local_id; + uint8_t owner_bit; + uint32_t pi; + + if (udma_jfc->mode != (uint32_t)UDMA_NORMAL_JFC_TYPE) + return; + + if (!jfc->jfc_cfg.flag.bs.lock_free) + spin_lock(&udma_jfc->lock); + + for (pi = udma_jfc->ci; get_next_cqe(udma_jfc, pi) != NULL; ++pi) { + if (pi > udma_jfc->ci + udma_jfc->buf.entry_cnt) + break; + } + while ((int) --pi - (int) udma_jfc->ci >= 0) { + cqe = get_buf_entry(&udma_jfc->buf, pi); + /* make sure cqe buffer is valid */ + rmb(); + local_id = (cqe->local_num_h << UDMA_SRC_IDX_SHIFT) | cqe->local_num_l; + if (local_id == jetty_id) { + if (cqe->s_r == CQE_FOR_RECEIVE) { + cr.local_id = local_id; + (void)udma_update_jfr_idx(udma_dev, cqe, &cr, true); + } + + ++nfreed; + } else if (!!nfreed) { + dest = get_buf_entry(&udma_jfc->buf, pi + nfreed); + /* make sure owner bit is valid */ + rmb(); + owner_bit = dest->owner; + (void)memcpy(dest, cqe, udma_dev->caps.cqe_size); + dest->owner = owner_bit; + } + } + + if (!!nfreed) { + udma_jfc->ci += nfreed; + wmb(); /* be sure software get cqe data before update doorbell */ + *udma_jfc->db.db_record = udma_jfc->ci & (uint32_t)UDMA_JFC_DB_CI_IDX_M; + } + + if (!jfc->jfc_cfg.flag.bs.lock_free) + spin_unlock(&udma_jfc->lock); +} diff --git a/drivers/ub/urma/hw/udma/udma_jfc.h b/drivers/ub/urma/hw/udma/udma_jfc.h new file mode 100644 index 0000000000000000000000000000000000000000..6f62f33eccdf49f77219b677064a509b8b9cafb3 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_jfc.h @@ -0,0 +1,198 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_JFC_H__ +#define __UDMA_JFC_H__ + +#include "udma_dev.h" +#include "udma_ctx.h" + +#define UDMA_JFC_DEPTH_MIN 64 +#define UDMA_JFC_DEPTH_SHIFT_BASE 6 + +#define CQE_VA_L_OFFSET 12 +#define CQE_VA_H_OFFSET 32 + +#define UDMA_DB_L_OFFSET 6 +#define UDMA_DB_H_OFFSET 38 + +#define UDMA_STARS_SWITCH 1 + +#define UDMA_JFC_DB_CI_IDX_M GENMASK(21, 0) +#define UDMA_CQE_INV_TOKEN_ID GENMASK(19, 0) + +enum udma_jfc_state { + UDMA_JFC_STATE_INVALID, + UDMA_JFC_STATE_VALID, + UDMA_JFC_STATE_ERROR, +}; + +enum udma_armed_jfc { + UDMA_CTX_NO_ARMED, + UDMA_CTX_ALWAYS_ARMED, + UDMA_CTX_REG_NEXT_CEQE, + UDMA_CTX_REG_NEXT_SOLICITED_CEQE, +}; + +enum udma_record_db { + UDMA_NO_RECORD_EN, + UDMA_RECORD_EN, +}; + +enum udma_cq_cnt_mode { + UDMA_CQE_CNT_MODE_BY_COUNT, + UDMA_CQE_CNT_MODE_BY_CI_PI_GAP, +}; + +struct udma_jfc { + struct ubcore_jfc base; + struct udma_context *ctx; + uint32_t jfcn; + uint32_t ceqn; + uint32_t tid; + struct udma_buf buf; + struct udma_sw_db db; + uint32_t ci; + uint32_t arm_sn; /* only kernel mode use */ + spinlock_t lock; + refcount_t event_refcount; + struct completion event_comp; + uint32_t lock_free; + uint32_t inline_en; + uint32_t mode; + uint64_t stars_chnl_addr; + bool stars_en; + uint32_t cq_shift; +}; + +struct udma_jfc_ctx { + /* DW0 */ + uint32_t state : 2; + uint32_t arm_st : 2; + uint32_t shift : 4; + uint32_t cqe_size : 1; + uint32_t record_db_en : 1; + uint32_t jfc_type : 1; + uint32_t inline_en : 1; + uint32_t cqe_va_l : 20; + /* DW1 */ + uint32_t cqe_va_h; + /* DW2 */ + uint32_t cqe_token_id : 20; + uint32_t cq_cnt_mode : 1; + uint32_t rsv0 : 3; + uint32_t ceqn : 8; + /* DW3 */ + uint32_t cqe_token_value : 24; + uint32_t rsv1 : 8; + /* DW4 */ + uint32_t pi : 22; + uint32_t cqe_coalesce_cnt : 10; + /* DW5 */ + uint32_t ci : 22; + uint32_t cqe_coalesce_period : 3; + uint32_t rsv2 : 7; + /* DW6 */ + uint32_t record_db_addr_l; + /* DW7 */ + uint32_t record_db_addr_h : 26; + uint32_t rsv3 : 6; + /* DW8 */ + uint32_t push_usi_en : 1; + uint32_t push_cqe_en : 1; + uint32_t token_en : 1; + uint32_t rsv4 : 9; + uint32_t tpn : 20; + /* DW9 ~ DW12 */ + uint32_t rmt_eid[4]; + /* DW13 */ + uint32_t seid_idx : 10; + uint32_t rmt_token_id : 20; + uint32_t rsv5 : 2; + /* DW14 */ + uint32_t remote_token_value; + /* DW15 */ + uint32_t int_vector : 16; + uint32_t stars_en : 1; + uint32_t rsv6 : 15; + /* DW16 */ + uint32_t poll : 1; + uint32_t cqe_report_timer : 24; + uint32_t se : 1; + uint32_t arm_sn : 2; + uint32_t rsv7 : 4; + /* DW17 */ + uint32_t se_cqe_idx : 24; + uint32_t rsv8 : 8; + /* DW18 */ + uint32_t wr_cqe_idx : 22; + uint32_t rsv9 : 10; + /* DW19 */ + uint32_t cqe_cnt : 24; + uint32_t rsv10 : 8; + /* DW20 ~ DW31 */ + uint32_t rsv11[12]; +}; + +struct udma_jfc_cqe { + /* DW0 */ + uint32_t s_r : 1; + uint32_t is_jetty : 1; + uint32_t owner : 1; + uint32_t inline_en : 1; + uint32_t opcode : 3; + uint32_t fd : 1; + uint32_t rsv : 8; + uint32_t substatus : 8; + uint32_t status : 8; + /* DW1 */ + uint32_t entry_idx : 16; + uint32_t local_num_l : 16; + /* DW2 */ + uint32_t local_num_h : 4; + uint32_t rmt_idx : 20; + uint32_t rsv1 : 8; + /* DW3 */ + uint32_t tpn : 24; + uint32_t rsv2 : 8; + /* DW4 */ + uint32_t byte_cnt; + /* DW5 ~ DW6 */ + uint32_t user_data_l; + uint32_t user_data_h; + /* DW7 ~ DW10 */ + uint32_t rmt_eid[4]; + /* DW11 ~ DW12 */ + uint32_t data_l; + uint32_t data_h; + /* DW13 ~ DW15 */ + uint32_t inline_data[3]; +}; + +struct udma_inv_tid { + uint32_t tid; + struct list_head list; +}; + +static inline struct udma_jfc *to_udma_jfc(struct ubcore_jfc *jfc) +{ + return container_of(jfc, struct udma_jfc, base); +} + +struct ubcore_jfc *udma_create_jfc(struct ubcore_device *ubcore_dev, + struct ubcore_jfc_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jfc(struct ubcore_jfc *jfc); +int udma_jfc_completion(struct notifier_block *nb, unsigned long jfcn, + void *data); +int udma_modify_jfc(struct ubcore_jfc *ubcore_jfc, struct ubcore_jfc_attr *attr, + struct ubcore_udata *udata); +int udma_rearm_jfc(struct ubcore_jfc *jfc, bool solicited_only); +int udma_poll_jfc(struct ubcore_jfc *jfc, int cr_cnt, struct ubcore_cr *cr); +int udma_check_jfc_cfg(struct udma_dev *dev, struct udma_jfc *jfc, + struct ubcore_jfc_cfg *cfg); +void udma_init_jfc_param(struct ubcore_jfc_cfg *cfg, struct udma_jfc *jfc); +int udma_post_create_jfc_mbox(struct udma_dev *dev, struct udma_jfc *jfc); +void udma_clean_jfc(struct ubcore_jfc *jfc, uint32_t jetty_id, struct udma_dev *udma_dev); + +#endif /* __UDMA_JFC_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_jfr.c b/drivers/ub/urma/hw/udma/udma_jfr.c new file mode 100644 index 0000000000000000000000000000000000000000..8e98319715e028cba2e0c1a5f1cbfbaffbf43a73 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_jfr.c @@ -0,0 +1,949 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt + +#include +#include "udma_cmd.h" +#include +#include "udma_jetty.h" +#include "udma_common.h" +#include "udma_db.h" +#include "udma_jfc.h" +#include "udma_jfr.h" + +const char *state_str[] = { + "RESET", + "READY", + "ERROR", + "INVALID" +}; + +static const char *to_state_str(enum ubcore_jfr_state state) +{ + if ((int)state >= (int)JFR_STATE_NUM) + return state_str[JFR_STATE_NUM]; + + return state_str[state]; +} + +static int udma_verify_jfr_param(struct udma_dev *dev, + struct ubcore_jfr_cfg *cfg) +{ + if (!cfg->max_sge || !cfg->depth || cfg->depth > dev->caps.jfr.depth || + cfg->max_sge > dev->caps.jfr_sge) { + dev_err(dev->dev, "Invalid jfr param, depth = %u, max_sge = %u.\n", + cfg->depth, cfg->max_sge); + return -EINVAL; + } + + if (cfg->flag.bs.token_policy > UBCORE_TOKEN_PLAIN_TEXT) { + dev_err(dev->dev, "jfr key policy = %d is not supported now.\n", + cfg->flag.bs.token_policy); + return -EINVAL; + } + + return 0; +} + +static int udma_get_k_jfr_buf(struct udma_dev *dev, struct udma_jfr *jfr) +{ + uint32_t idx_buf_size; + int ret; + + jfr->rq.buf.entry_size = UDMA_SGE_SIZE * min(jfr->max_sge, dev->caps.jfr_sge); + jfr->rq.buf.entry_cnt = jfr->wqe_cnt; + ret = udma_k_alloc_buf(dev, &jfr->rq.buf); + if (ret) { + dev_err(dev->dev, "failed to alloc rq buffer, id=%u.\n", jfr->rq.id); + return ret; + } + + jfr->idx_que.buf.entry_size = UDMA_IDX_QUE_ENTRY_SZ; + jfr->idx_que.buf.entry_cnt = jfr->wqe_cnt; + idx_buf_size = jfr->idx_que.buf.entry_size * jfr->idx_que.buf.entry_cnt; + ret = udma_alloc_normal_buf(dev, idx_buf_size, &jfr->idx_que.buf); + if (ret) { + dev_err(dev->dev, + "failed to alloc idx que buffer for jfr when buffer size = %u.\n", + idx_buf_size); + goto err_idx_que; + } + + jfr->rq.wrid = kcalloc(1, jfr->rq.buf.entry_cnt * sizeof(uint64_t), GFP_KERNEL); + if (!jfr->rq.wrid) + goto err_wrid; + + jfr->jetty_addr = (uintptr_t)&jfr->rq; + + if (udma_alloc_sw_db(dev, &jfr->sw_db, UDMA_JFR_TYPE_DB)) { + dev_err(dev->dev, "failed to alloc sw db for jfr(%u).\n", jfr->rq.id); + goto err_alloc_db; + } + + udma_init_udma_table(&jfr->idx_que.jfr_idx_table, jfr->idx_que.buf.entry_cnt - 1, 0); + + jfr->rq.tid = dev->tid; + + return 0; + +err_alloc_db: + kfree(jfr->rq.wrid); +err_wrid: + udma_free_normal_buf(dev, idx_buf_size, &jfr->idx_que.buf); +err_idx_que: + udma_k_free_buf(dev, &jfr->rq.buf); + + return -ENOMEM; +} + +static int udma_jfr_get_u_cmd(struct udma_dev *dev, struct ubcore_udata *udata, + struct udma_create_jetty_ucmd *ucmd) +{ + unsigned long byte; + + if (!udata->udrv_data) { + dev_err(dev->dev, "jfr udata udrv_data is null.\n"); + return -EINVAL; + } + + if (!udata->udrv_data->in_addr || udata->udrv_data->in_len != sizeof(*ucmd)) { + dev_err(dev->dev, "jfr in_len %u or addr is invalid.\n", + udata->udrv_data->in_len); + return -EINVAL; + } + + byte = copy_from_user(ucmd, (void *)(uintptr_t)udata->udrv_data->in_addr, + sizeof(*ucmd)); + if (byte) { + dev_err(dev->dev, + "failed to copy jfr udata, byte = %lu.\n", byte); + return -EFAULT; + } + + return 0; +} + +static int udma_get_u_jfr_buf(struct udma_dev *dev, struct udma_jfr *jfr, + struct ubcore_udata *udata, + struct udma_create_jetty_ucmd *ucmd) +{ + int ret; + + ret = udma_jfr_get_u_cmd(dev, udata, ucmd); + if (ret) + return ret; + + jfr->udma_ctx = to_udma_context(udata->uctx); + if (ucmd->non_pin) { + jfr->rq.buf.addr = ucmd->buf_addr; + } else if (ucmd->is_hugepage) { + jfr->rq.buf.addr = ucmd->buf_addr; + if (udma_occupy_u_hugepage(jfr->udma_ctx, (void *)jfr->rq.buf.addr)) { + dev_err(dev->dev, "failed to create rq, va not map.\n"); + return -EINVAL; + } + jfr->rq.buf.is_hugepage = true; + } else { + ret = pin_queue_addr(dev, ucmd->buf_addr, ucmd->buf_len, &jfr->rq.buf); + if (ret) { + dev_err(dev->dev, + "failed to pin jfr rqe buf addr, ret = %d.\n", ret); + return ret; + } + } + + if (ucmd->non_pin) { + jfr->idx_que.buf.addr = ucmd->idx_addr; + } else { + ret = pin_queue_addr(dev, ucmd->idx_addr, ucmd->idx_len, + &jfr->idx_que.buf); + if (ret) { + dev_err(dev->dev, + "failed to pin jfr idx que addr, ret = %d.\n", ret); + goto err_pin_idx_buf; + } + } + + jfr->sw_db.db_addr = ucmd->db_addr; + jfr->jfr_sleep_buf.db_addr = ucmd->jfr_sleep_buf; + + if (!ucmd->non_pin) { + ret = udma_pin_sw_db(jfr->udma_ctx, &jfr->sw_db); + if (ret) { + dev_err(dev->dev, + "failed to pin jfr sw db addr, ret = %d.\n", ret); + goto err_pin_sw_db; + } + + ret = udma_pin_sw_db(jfr->udma_ctx, &jfr->jfr_sleep_buf); + if (ret) { + dev_err(dev->dev, + "failed to pin jfr sleep time buf, ret = %d.\n", ret); + goto err_pin_jfr_sleep_buf; + } + } + + jfr->jetty_addr = ucmd->jetty_addr; + jfr->rq.tid = jfr->udma_ctx->tid; + + return ret; + +err_pin_jfr_sleep_buf: + udma_unpin_sw_db(jfr->udma_ctx, &jfr->sw_db); +err_pin_sw_db: + unpin_queue_addr(jfr->idx_que.buf.umem); +err_pin_idx_buf: + if (ucmd->is_hugepage) + udma_return_u_hugepage(jfr->udma_ctx, (void *)jfr->rq.buf.addr); + else + unpin_queue_addr(jfr->rq.buf.umem); + return ret; +} + +static int udma_get_jfr_buf(struct udma_dev *dev, struct udma_jfr *jfr, + struct ubcore_udata *udata) +{ + struct udma_create_jetty_ucmd ucmd = {}; + + if (udata == NULL) + return udma_get_k_jfr_buf(dev, jfr); + else + return udma_get_u_jfr_buf(dev, jfr, udata, &ucmd); +} + +static void udma_put_jfr_buf(struct udma_dev *dev, struct udma_jfr *jfr) +{ + uint32_t size; + + if (!jfr->rq.buf.kva && !jfr->idx_que.buf.kva && + jfr->sw_db.page && jfr->jfr_sleep_buf.page) { + udma_unpin_sw_db(jfr->udma_ctx, &jfr->jfr_sleep_buf); + udma_unpin_sw_db(jfr->udma_ctx, &jfr->sw_db); + unpin_queue_addr(jfr->idx_que.buf.umem); + if (jfr->rq.buf.is_hugepage) + udma_return_u_hugepage(jfr->udma_ctx, (void *)jfr->rq.buf.addr); + else + unpin_queue_addr(jfr->rq.buf.umem); + return; + } + + if (jfr->rq.buf.kva) { + udma_k_free_buf(dev, &jfr->rq.buf); + udma_free_sw_db(dev, &jfr->sw_db); + } + + if (jfr->idx_que.buf.kva) { + size = jfr->idx_que.buf.entry_cnt * jfr->idx_que.buf.entry_size; + udma_free_normal_buf(dev, size, &jfr->idx_que.buf); + udma_destroy_udma_table(dev, &jfr->idx_que.jfr_idx_table, "JFR_IDX"); + } + + kfree(jfr->rq.wrid); +} + +static enum udma_rx_limit_wl to_udma_limit_wl(uint32_t rx_threshold) +{ + if (rx_threshold >= LIMIT_WL_4096_V) + return UDMA_RX_LIMIT_WL_4096; + if (rx_threshold >= LIMIT_WL_512_V) + return UDMA_RX_LIMIT_WL_512; + if (rx_threshold >= LIMIT_WL_64_V) + return UDMA_RX_LIMIT_WL_64; + + return UDMA_RX_LIMIT_WL_0; +} + +static void udma_init_jfrc(struct udma_dev *dev, struct ubcore_jfr_cfg *cfg, + struct udma_jfr *jfr, void *mb_buf, + uint32_t rx_threshold) +{ + struct udma_jfr_ctx *ctx = (struct udma_jfr_ctx *)mb_buf; + struct udma_jfc *jfc = to_udma_jfc(cfg->jfc); + uint32_t tid = jfr->rq.tid; + uint64_t db_addr; + + db_addr = jfr->sw_db.db_addr; + + memset(ctx, 0, sizeof(struct udma_jfr_ctx) * UDMA_CTX_NUM); + ctx->state = UDMA_JFR_STATE_READY; + ctx->record_db_en = 1; + ctx->rqe_base_addr_l = (jfr->rq.buf.addr >> RQE_VA_L_PAGE_4K_OFFSET) & + (uint32_t)RQE_VA_L_VALID_BIT; + ctx->rqe_base_addr_h = (jfr->rq.buf.addr >> (uint32_t)RQE_VA_H_PAGE_4K_OFFSET) & + (uint32_t)RQE_VA_H_VALID_BIT; + ctx->idx_que_addr_l = (jfr->idx_que.buf.addr >> JFR_IDX_VA_L_PAGE_4K_OFFSET) & + (uint32_t)JFR_IDX_VA_L_VALID_BIT; + ctx->idx_que_addr_h = (jfr->idx_que.buf.addr >> (uint32_t)JFR_IDX_VA_H_PAGE_4K_OFFSET) & + (uint32_t)JFR_IDX_VA_H_VALID_BIT; + ctx->record_db_addr_l = (db_addr >> JFR_DB_VA_L_PAGE_64_OFFSET) & + (uint32_t)JFR_DB_VA_L_VALID_BIT; + ctx->record_db_addr_m = (db_addr >> (uint32_t)JFR_DB_VA_M_PAGE_64_OFFSET) & + (uint32_t)JFR_DB_VA_M_VALID_BIT; + ctx->record_db_addr_h = (db_addr >> (uint32_t)JFR_DB_VA_H_PAGE_64_OFFSET) & + (uint32_t)JFR_DB_VA_H_VALID_BIT; + ctx->rqe_token_id_l = tid & (uint32_t)RQE_TOKEN_ID_L_MASK; + ctx->rqe_token_id_h = (tid >> RQE_TOKEN_ID_H_OFFSET) & (uint32_t)RQE_TOKEN_ID_H_MASK; + ctx->jfcn_l = cfg->jfc->id & (uint32_t)JFR_JFCN_L_VALID_BIT; + ctx->jfcn_h = (cfg->jfc->id >> JFR_JFCN_H_OFFSET) & (uint32_t)JFR_JFCN_H_VALID_BIT; + if (cfg->min_rnr_timer > UDMA_RNR_MAX) { + dev_warn(dev->dev, + "min_rnr_timer is out of range, max_value(%d) is applied.\n", + UDMA_RNR_MAX); + ctx->rnr_timer = UDMA_RNR_MAX; + } else { + ctx->rnr_timer = cfg->min_rnr_timer; + } + if (cfg->flag.bs.token_policy != UBCORE_TOKEN_NONE) + ctx->token_en = 1; + ctx->type = to_udma_type(cfg->trans_mode); + ctx->token_value = cfg->token_value.token; + ctx->user_data_l = jfr->jetty_addr; + ctx->user_data_h = jfr->jetty_addr >> UDMA_USER_DATA_H_OFFSET; + ctx->rqe_size_shift = ilog2(jfr->max_sge); + ctx->rqe_shift = ilog2(jfr->wqe_cnt); + if (!!(dev->caps.feature & UDMA_CAP_FEATURE_JFC_INLINE)) + ctx->cqeie = jfc->inline_en; + + ctx->limit_wl = (uint32_t)to_udma_limit_wl(rx_threshold); + ctx->pld_token_id = tid & (uint32_t)JFR_PLD_TOKEN_ID_MASK; +} + +static void udma_reset_sw_k_jfr_queue(struct udma_jfr *jfr) +{ + ida_destroy(&jfr->idx_que.jfr_idx_table.ida_table.ida); + ida_init(&jfr->idx_que.jfr_idx_table.ida_table.ida); + jfr->rq.pi = 0; + jfr->rq.ci = 0; + *jfr->sw_db.db_record = 0; +} + +static int udma_hw_init_jfrc(struct udma_dev *dev, struct ubcore_jfr_cfg *cfg, + struct udma_jfr *jfr, uint32_t rx_threshold) +{ + struct ubase_mbx_attr mbox_attr = {}; + struct ubase_cmd_mailbox *mailbox; + struct udma_jfr_ctx *ctx = NULL; + int ret; + + mailbox = udma_alloc_cmd_mailbox(dev); + if (!mailbox) { + dev_err(dev->dev, "failed to alloc mailbox for JFRC.\n"); + return -ENOMEM; + } + + udma_init_jfrc(dev, cfg, jfr, mailbox->buf, rx_threshold); + + mbox_attr.tag = jfr->rq.id; + mbox_attr.op = UDMA_CMD_CREATE_JFR_CONTEXT; + ret = udma_post_mbox(dev, mailbox, &mbox_attr); + if (ret) + dev_err(dev->dev, + "failed to post mbox cmd of create JFRC, ret = %d.\n", + ret); + + if (jfr->rq.buf.kva) + udma_reset_sw_k_jfr_queue(jfr); + + ctx = (struct udma_jfr_ctx *)mailbox->buf; + ctx->token_value = 0; + udma_free_cmd_mailbox(dev, mailbox); + + return ret; +} + +static void set_jfr_param(struct udma_jfr *jfr, struct ubcore_jfr_cfg *cfg) +{ + if (cfg->depth < UDMA_MIN_JFR_DEPTH) + jfr->wqe_cnt = UDMA_MIN_JFR_DEPTH; + else + jfr->wqe_cnt = roundup_pow_of_two(cfg->depth); + + jfr->ubcore_jfr.jfr_id.id = jfr->rq.id; + jfr->ubcore_jfr.jfr_cfg = *cfg; + jfr->max_sge = roundup_pow_of_two(cfg->max_sge); + jfr->ubcore_jfr.jfr_cfg.max_sge = jfr->max_sge; + jfr->ubcore_jfr.jfr_cfg.depth = jfr->wqe_cnt; + jfr->state = UBCORE_JFR_STATE_READY; + + if (!cfg->flag.bs.lock_free) + spin_lock_init(&jfr->lock); +} + +static int udma_alloc_jfr_id(struct udma_dev *udma_dev, uint32_t cfg_id, uint32_t *idx) +{ + struct udma_ida *ida_table = &udma_dev->jfr_table.ida_table; + uint32_t min; + uint32_t max; + int id; + + if (cfg_id && (cfg_id < ida_table->min || cfg_id > ida_table->max)) { + dev_err(udma_dev->dev, + "user specify id %u error, min %u max %u.\n", + cfg_id, ida_table->min, ida_table->max); + return -EINVAL; + } + + spin_lock(&ida_table->lock); + min = cfg_id ? cfg_id : ida_table->next; + max = cfg_id ? cfg_id : ida_table->max; + id = ida_alloc_range(&ida_table->ida, min, max, GFP_ATOMIC); + if (id < 0) { + if (!cfg_id) + id = ida_alloc_range(&ida_table->ida, min = ida_table->min, + max, GFP_ATOMIC); + if (id < 0) { + dev_err(udma_dev->dev, + "alloc jfr id range (%u - %u) failed, ret = %d.\n", + min, max, id); + spin_unlock(&ida_table->lock); + + return id; + } + } + + *idx = (uint32_t)id; + + if (!cfg_id) + ida_table->next = (uint32_t)id + 1 > ida_table->max ? + ida_table->min : (uint32_t)id + 1; + spin_unlock(&ida_table->lock); + + return 0; +} + +struct ubcore_jfr *udma_create_jfr(struct ubcore_device *dev, + struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_jfr *udma_jfr; + int ret; + + ret = udma_verify_jfr_param(udma_dev, cfg); + if (ret) { + dev_err(udma_dev->dev, "verify jfr param failed.\n"); + return NULL; + } + + udma_jfr = kzalloc(sizeof(*udma_jfr), GFP_KERNEL); + if (!udma_jfr) + return NULL; + + ret = udma_alloc_jfr_id(udma_dev, cfg->id, &udma_jfr->rq.id); + if (ret) + goto err_alloc_jfr_id; + + set_jfr_param(udma_jfr, cfg); + + ret = udma_get_jfr_buf(udma_dev, udma_jfr, udata); + if (ret) + goto err_get_jfr_buf; + + ret = xa_err(xa_store(&udma_dev->jfr_table.xa, udma_jfr->rq.id, + udma_jfr, GFP_KERNEL)); + if (ret) { + dev_err(udma_dev->dev, "store jfr to jfr_table failed.\n"); + goto err_xa_store; + } + + ret = udma_hw_init_jfrc(udma_dev, cfg, udma_jfr, 0); + if (ret) { + dev_err(udma_dev->dev, "failed to init JFRC, ret = %d.\n", ret); + goto err_hw_init_jfrc; + } + + refcount_set(&udma_jfr->ae_refcount, 1); + init_completion(&udma_jfr->ae_comp); + + if (dfx_switch) + udma_dfx_store_id(udma_dev, &udma_dev->dfx_info->jfr, udma_jfr->rq.id, "jfr"); + + return &udma_jfr->ubcore_jfr; + +err_hw_init_jfrc: + xa_erase(&udma_dev->jfr_table.xa, udma_jfr->rq.id); +err_xa_store: + udma_put_jfr_buf(udma_dev, udma_jfr); +err_get_jfr_buf: + udma_id_free(&udma_dev->jfr_table.ida_table, udma_jfr->rq.id); +err_alloc_jfr_id: + kfree(udma_jfr); + return NULL; +} + +static int modify_jfr_context(struct udma_dev *dev, uint32_t jfrn, + bool state_flag, bool rx_threshold_flag, + struct ubcore_jfr_attr *attr) +{ + struct ubase_mbx_attr mbox_attr = {}; + struct udma_jfr_ctx *ctx, *ctx_mask; + struct ubase_cmd_mailbox *mailbox; + int ret; + + mailbox = udma_alloc_cmd_mailbox(dev); + if (!mailbox) { + dev_err(dev->dev, "failed to alloc mailbox for JFRC.\n"); + return -EINVAL; + } + + ctx = (struct udma_jfr_ctx *)mailbox->buf; + ctx_mask = ctx + 1; + memset(ctx_mask, 0xff, sizeof(struct udma_jfr_ctx)); + if (state_flag) { + ctx->state = attr->state; + ctx_mask->state = 0; + } + + if (rx_threshold_flag) { + ctx->limit_wl = (uint32_t)to_udma_limit_wl(attr->rx_threshold); + ctx_mask->limit_wl = 0; + } + + mbox_attr.tag = jfrn; + mbox_attr.op = UDMA_CMD_MODIFY_JFR_CONTEXT; + + ret = udma_post_mbox(dev, mailbox, &mbox_attr); + if (ret) + dev_err(dev->dev, + "failed to post mbox cmd of modify JFRC, ret = %d.\n", ret); + + udma_free_cmd_mailbox(dev, mailbox); + + return ret; +} + +static int udma_modify_jfr_to_error(struct ubcore_jfr *jfr, bool *need_sleep) +{ + struct udma_dev *udma_dev = to_udma_dev(jfr->ub_dev); + struct udma_jfr *udma_jfr = to_udma_jfr(jfr); + struct ubcore_jfr_attr attr; + int ret = 0; + + if (udma_jfr->state == UBCORE_JFR_STATE_READY) { + attr.state = UBCORE_JFR_STATE_ERROR; + attr.mask = UBCORE_JFR_STATE; + ret = modify_jfr_context(udma_dev, udma_jfr->rq.id, true, false, &attr); + if (ret) { + dev_err(udma_dev->dev, "failed to modify jfr state to error, id: %u.\n", + udma_jfr->rq.id); + return ret; + } + + udma_jfr->state = UBCORE_JFR_STATE_ERROR; + + *need_sleep = true; + } + + return ret; +} + +static int udma_modify_jfr_to_reset(struct ubcore_jfr *jfr) +{ + struct udma_dev *udma_dev = to_udma_dev(jfr->ub_dev); + struct udma_jfr *udma_jfr = to_udma_jfr(jfr); + struct ubase_mbx_attr mbox_attr = {}; + int ret = 0; + + if (udma_jfr->state != UBCORE_JFR_STATE_RESET) { + mbox_attr.tag = udma_jfr->rq.id; + mbox_attr.op = UDMA_CMD_DESTROY_JFR_CONTEXT; + ret = post_mailbox_update_ctx(udma_dev, NULL, 0, &mbox_attr); + if (ret) { + dev_err(udma_dev->dev, "failed to post jfr destroy cmd, id: %u.\n", + udma_jfr->rq.id); + return ret; + } + + udma_jfr->state = UBCORE_JFR_STATE_RESET; + } + + return ret; +} + +static int udma_modify_and_del_jfr(struct udma_dev *udma_dev, struct udma_jfr *udma_jfr) +{ + bool large_payload = false; + bool need_sleep = false; + uint32_t sleep_time = 0; + int ret = 0; + + ret = udma_modify_jfr_to_error(&udma_jfr->ubcore_jfr, &need_sleep); + if (ret) + return ret; + if (!udma_jfr->rq.buf.kva && udma_jfr->jfr_sleep_buf.page) + large_payload = !!(*(bool *)udma_jfr->jfr_sleep_buf.virt_addr); + if (need_sleep) { + sleep_time = large_payload ? jfr_sleep_time : UDMA_DEF_JFR_SLEEP_TIME; + dev_info_ratelimited(udma_dev->dev, "jfr sleep time = %u us.\n", sleep_time); + usleep_range(sleep_time, sleep_time + UDMA_SLEEP_DELAY_TIME); + } + + return udma_modify_jfr_to_reset(&udma_jfr->ubcore_jfr); +} + +static void udma_free_jfr(struct ubcore_jfr *jfr) +{ + struct udma_dev *udma_dev = to_udma_dev(jfr->ub_dev); + struct udma_jfr *udma_jfr = to_udma_jfr(jfr); + + if (udma_jfr->rq.buf.kva && jfr->jfr_cfg.jfc) + udma_clean_jfc(jfr->jfr_cfg.jfc, udma_jfr->rq.id, udma_dev); + + if (dfx_switch) + udma_dfx_delete_id(udma_dev, &udma_dev->dfx_info->jfr, udma_jfr->rq.id); + + xa_erase(&udma_dev->jfr_table.xa, udma_jfr->rq.id); + + if (refcount_dec_and_test(&udma_jfr->ae_refcount)) + complete(&udma_jfr->ae_comp); + wait_for_completion(&udma_jfr->ae_comp); + + udma_put_jfr_buf(udma_dev, udma_jfr); + udma_id_free(&udma_dev->jfr_table.ida_table, udma_jfr->rq.id); + jfr->jfr_cfg.token_value.token = 0; + kfree(udma_jfr); +} + +int udma_destroy_jfr(struct ubcore_jfr *jfr) +{ + struct udma_dev *udma_dev = to_udma_dev(jfr->ub_dev); + struct udma_jfr *udma_jfr = to_udma_jfr(jfr); + int ret; + + ret = udma_modify_and_del_jfr(udma_dev, udma_jfr); + if (ret) { + dev_err(udma_dev->dev, + "failed to modify and delete jfr, id: %u, ret = %d.\n", + udma_jfr->rq.id, ret); + return ret; + } + + udma_free_jfr(jfr); + + return 0; +} + +int udma_destroy_jfr_batch(struct ubcore_jfr **jfr, int jfr_cnt, int *bad_jfr_index) +{ + bool large_payload = false; + struct udma_dev *udma_dev; + struct udma_jfr *udma_jfr; + bool need_sleep = false; + uint32_t sleep_time = 0; + uint32_t i; + int ret; + + if (!jfr) { + pr_info("jfr array is null.\n"); + return -EINVAL; + } + + if (!jfr_cnt) { + pr_info("jfr cnt is 0.\n"); + return -EINVAL; + } + + udma_dev = to_udma_dev(jfr[0]->ub_dev); + + for (i = 0; i < jfr_cnt; i++) { + ret = udma_modify_jfr_to_error(jfr[i], &need_sleep); + if (ret) { + *bad_jfr_index = 0; + return ret; + } + + if (unlikely(large_payload)) + continue; + udma_jfr = to_udma_jfr(jfr[i]); + if (!udma_jfr->rq.buf.kva && udma_jfr->jfr_sleep_buf.page) + large_payload = !!(*(bool *)udma_jfr->jfr_sleep_buf.virt_addr); + } + + if (need_sleep) { + sleep_time = large_payload ? jfr_sleep_time : UDMA_DEF_JFR_SLEEP_TIME; + dev_info(udma_dev->dev, "jfr sleep time = %u us.\n", sleep_time); + usleep_range(sleep_time, sleep_time + UDMA_SLEEP_DELAY_TIME); + } + + for (i = 0; i < jfr_cnt; i++) { + ret = udma_modify_jfr_to_reset(jfr[i]); + if (ret) { + *bad_jfr_index = 0; + return ret; + } + } + + for (i = 0; i < jfr_cnt; i++) + udma_free_jfr(jfr[i]); + + return 0; +} + +static bool verify_modify_jfr_state(enum ubcore_jfr_state jfr_state, + enum ubcore_jfr_state attr_state) +{ + switch (jfr_state) { + case UBCORE_JFR_STATE_RESET: + return attr_state == UBCORE_JFR_STATE_READY; + case UBCORE_JFR_STATE_READY: + return attr_state == UBCORE_JFR_STATE_ERROR; + case UBCORE_JFR_STATE_ERROR: + return attr_state == UBCORE_JFR_STATE_RESET; + default: + break; + } + + return false; +} + +static int verify_modify_jfr(struct udma_dev *udma_dev, struct udma_jfr *udma_jfr, + struct ubcore_jfr_attr *attr, bool *state_flag, + bool *rx_threshold_flag) +{ + *rx_threshold_flag = false; + *state_flag = false; + + if (!(attr->mask & (UBCORE_JFR_RX_THRESHOLD | UBCORE_JFR_STATE))) { + dev_err(udma_dev->dev, + "modify jfr mask is error or not set, jfrn = %u.\n", + udma_jfr->rq.id); + return -EINVAL; + } + + if (attr->mask & UBCORE_JFR_RX_THRESHOLD) { + if (attr->rx_threshold >= udma_jfr->wqe_cnt) { + dev_err(udma_dev->dev, + "JFR rx_threshold(%u) must less than wqe num(%u).\n", + attr->rx_threshold, udma_jfr->wqe_cnt); + return -EINVAL; + } + *rx_threshold_flag = true; + } + + if (attr->mask & UBCORE_JFR_STATE) { + if (udma_jfr->state == attr->state) { + dev_info(udma_dev->dev, + "jfr(%u) state has been %s, keep it unchanged.\n", + udma_jfr->rq.id, to_state_str(attr->state)); + return 0; + } else if (!verify_modify_jfr_state(udma_jfr->state, + attr->state)) { + dev_err(udma_dev->dev, + "jfr(%u) not support modify jfr state from %s to %s.\n", + udma_jfr->rq.id, to_state_str(udma_jfr->state), + to_state_str(attr->state)); + return -EINVAL; + } else if ((attr->state == UBCORE_JFR_STATE_RESET || + attr->state == UBCORE_JFR_STATE_ERROR) && + *rx_threshold_flag) { + dev_err(udma_dev->dev, + "jfr(%u) not support set rx threshold when change state to %s.\n", + udma_jfr->rq.id, to_state_str(attr->state)); + return -EINVAL; + } + *state_flag = true; + } + + return 0; +} + +static int udma_destroy_hw_jfr_ctx(struct udma_dev *dev, uint32_t jfr_id) +{ + struct ubase_mbx_attr attr = {}; + int ret; + + attr.tag = jfr_id; + attr.op = UDMA_CMD_DESTROY_JFR_CONTEXT; + ret = post_mailbox_update_ctx(dev, NULL, 0, &attr); + if (ret) + dev_err(dev->dev, + "post mailbox destroy jfr ctx failed, ret = %d.\n", ret); + + return ret; +} + +int udma_modify_jfr(struct ubcore_jfr *jfr, struct ubcore_jfr_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(jfr->ub_dev); + struct udma_jfr *udma_jfr = to_udma_jfr(jfr); + bool rx_threshold_flag = false; + bool state_flag = false; + int ret = 0; + + ret = verify_modify_jfr(udma_dev, udma_jfr, attr, &state_flag, + &rx_threshold_flag); + if (ret) + return ret; + + if (!(rx_threshold_flag || state_flag)) + return 0; + + if (rx_threshold_flag && !state_flag) { + ret = modify_jfr_context(udma_dev, udma_jfr->rq.id, state_flag, + rx_threshold_flag, attr); + } else { + switch (attr->state) { + case UBCORE_JFR_STATE_RESET: + ret = udma_destroy_hw_jfr_ctx(udma_dev, udma_jfr->rq.id); + break; + case UBCORE_JFR_STATE_READY: + ret = udma_hw_init_jfrc(udma_dev, &jfr->jfr_cfg, udma_jfr, + rx_threshold_flag ? + attr->rx_threshold : udma_jfr->rx_threshold); + break; + default: + ret = modify_jfr_context(udma_dev, udma_jfr->rq.id, state_flag, + rx_threshold_flag, attr); + break; + } + } + + if (ret) + return ret; + + if (state_flag) + udma_jfr->state = attr->state; + + if (rx_threshold_flag) + udma_jfr->rx_threshold = attr->rx_threshold; + + return 0; +} + +int udma_unimport_jfr(struct ubcore_tjetty *tjfr) +{ + struct udma_target_jetty *udma_tjfr = to_udma_tjetty(tjfr); + + udma_tjfr->token_value = 0; + tjfr->cfg.token_value.token = 0; + + kfree(udma_tjfr); + + return 0; +} + +static void fill_wqe_idx(struct udma_jfr *jfr, uint32_t wqe_idx) +{ + uint32_t *idx_buf; + + idx_buf = (uint32_t *)get_buf_entry(&jfr->idx_que.buf, jfr->rq.pi); + *idx_buf = cpu_to_le32(wqe_idx); + + jfr->rq.pi++; +} + +static void fill_recv_sge_to_wqe(struct ubcore_jfr_wr *wr, void *wqe, + uint32_t max_sge) +{ + struct udma_wqe_sge *sge = (struct udma_wqe_sge *)wqe; + uint32_t i, cnt; + + for (i = 0, cnt = 0; i < wr->src.num_sge; i++) { + if (!wr->src.sge[i].len) + continue; + set_data_of_sge(sge + cnt, wr->src.sge + i); + ++cnt; + } + + if (cnt < max_sge) + memset(sge + cnt, 0, (max_sge - cnt) * UDMA_SGE_SIZE); +} + +static int post_recv_one(struct udma_dev *dev, struct udma_jfr *jfr, + struct ubcore_jfr_wr *wr) +{ + uint32_t wqe_idx; + int ret = 0; + void *wqe; + + if (unlikely(wr->src.num_sge > jfr->max_sge)) { + dev_err(dev->dev, + "failed to check sge, wr_num_sge = %u, max_sge = %u, jfrn = %u.\n", + wr->src.num_sge, jfr->max_sge, jfr->rq.id); + return -EINVAL; + } + + if (udma_jfrwq_overflow(jfr)) { + dev_err(dev->dev, "failed to check jfrwq, jfrwq is full, jfrn = %u.\n", + jfr->rq.id); + return -ENOMEM; + } + + ret = udma_id_alloc(dev, &jfr->idx_que.jfr_idx_table.ida_table, + &wqe_idx); + if (ret) { + dev_err(dev->dev, "failed to get jfr wqe idx.\n"); + return ret; + } + wqe = get_buf_entry(&jfr->rq.buf, wqe_idx); + + fill_recv_sge_to_wqe(wr, wqe, jfr->max_sge); + + fill_wqe_idx(jfr, wqe_idx); + + jfr->rq.wrid[wqe_idx] = wr->user_ctx; + + return ret; +} + +/* thanks to drivers/infiniband/hw/bnxt_re/ib_verbs.c */ +int udma_post_jfr_wr(struct ubcore_jfr *ubcore_jfr, struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr) +{ + struct udma_dev *dev = to_udma_dev(ubcore_jfr->ub_dev); + struct udma_jfr *jfr = to_udma_jfr(ubcore_jfr); + unsigned long flags; + uint32_t nreq; + int ret = 0; + + if (!ubcore_jfr->jfr_cfg.flag.bs.lock_free) + spin_lock_irqsave(&jfr->lock, flags); + + for (nreq = 0; wr; ++nreq, wr = wr->next) { + ret = post_recv_one(dev, jfr, wr); + if (ret) { + *bad_wr = wr; + break; + } + } + + if (likely(nreq)) { + /* + * Ensure that the pipeline fills all RQEs into the RQ queue, + * then updating the PI pointer. + */ + wmb(); + *jfr->sw_db.db_record = jfr->rq.pi & + (uint32_t)UDMA_JFR_DB_PI_M; + } + + if (!ubcore_jfr->jfr_cfg.flag.bs.lock_free) + spin_unlock_irqrestore(&jfr->lock, flags); + + return ret; +} + +struct ubcore_tjetty *udma_import_jfr_ex(struct ubcore_device *dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_active_tp_cfg *active_tp_cfg, + struct ubcore_udata *udata) +{ + struct udma_target_jetty *udma_tjfr; + + udma_tjfr = kzalloc(sizeof(*udma_tjfr), GFP_KERNEL); + if (!udma_tjfr) + return NULL; + + if (!udata) { + if (cfg->flag.bs.token_policy != UBCORE_TOKEN_NONE) { + udma_tjfr->token_value = cfg->token_value.token; + udma_tjfr->token_value_valid = true; + } + } + + udma_swap_endian(cfg->id.eid.raw, udma_tjfr->le_eid.raw, UBCORE_EID_SIZE); + + return &udma_tjfr->ubcore_tjetty; +} diff --git a/drivers/ub/urma/hw/udma/udma_jfr.h b/drivers/ub/urma/hw/udma/udma_jfr.h new file mode 100644 index 0000000000000000000000000000000000000000..c446eaedee1db5fc95fe89cf3834fc2358942e1a --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_jfr.h @@ -0,0 +1,189 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_JFR_H__ +#define __UDMA_JFR_H__ + +#include "udma_dev.h" +#include "udma_ctx.h" +#include "udma_common.h" + +#define RQE_VA_L_PAGE_4K_OFFSET 12U +#define RQE_VA_L_VALID_BIT GENMASK(19, 0) +#define RQE_VA_H_OFFSET 20 +#define RQE_VA_H_PAGE_4K_OFFSET (RQE_VA_H_OFFSET + RQE_VA_L_PAGE_4K_OFFSET) +#define RQE_VA_H_VALID_BIT GENMASK(31, 0) + +#define RQE_TOKEN_ID_L_MASK GENMASK(13, 0) +#define RQE_TOKEN_ID_H_OFFSET 14U +#define RQE_TOKEN_ID_H_MASK GENMASK(5, 0) + +#define JFR_IDX_VA_L_PAGE_4K_OFFSET 12U +#define JFR_IDX_VA_L_VALID_BIT GENMASK(31, 0) +#define JFR_IDX_VA_H_OFFSET 32 +#define JFR_IDX_VA_H_PAGE_4K_OFFSET \ + (JFR_IDX_VA_H_OFFSET + JFR_IDX_VA_L_PAGE_4K_OFFSET) +#define JFR_IDX_VA_H_VALID_BIT GENMASK(19, 0) + +#define JFR_DB_VA_L_PAGE_64_OFFSET 6U +#define JFR_DB_VA_L_VALID_BIT GENMASK(23, 0) +#define JFR_DB_VA_M_OFFSET 24 +#define JFR_DB_VA_M_PAGE_64_OFFSET \ + (JFR_DB_VA_M_OFFSET + JFR_DB_VA_L_PAGE_64_OFFSET) +#define JFR_DB_VA_M_VALID_BIT GENMASK(31, 0) +#define JFR_DB_VA_H_OFFSET 32 +#define JFR_DB_VA_H_PAGE_64_OFFSET \ + (JFR_DB_VA_H_OFFSET + JFR_DB_VA_M_PAGE_64_OFFSET) +#define JFR_DB_VA_H_VALID_BIT GENMASK(1, 0) + +#define JFR_JFCN_L_VALID_BIT GENMASK(11, 0) +#define JFR_JFCN_H_OFFSET 12U +#define JFR_JFCN_H_VALID_BIT GENMASK(7, 0) + +#define UDMA_JFR_DB_PI_M GENMASK(15, 0) + +#define JFR_PLD_TOKEN_ID_MASK GENMASK(19, 0) + +#define UDMA_MIN_JFR_DEPTH 64 +#define UDMA_SGE_SIZE 16U +#define UDMA_IDX_QUE_ENTRY_SZ 4 +#define UDMA_RNR_MAX 19 + +#define UDMA_DEF_JFR_SLEEP_TIME 1000 +#define UDMA_SLEEP_DELAY_TIME 10 + +enum jfr_state { + UDMA_JFR_STATE_RESET = 0, + UDMA_JFR_STATE_READY, + UDMA_JFR_STATE_ERROR, + JFR_STATE_NUM, +}; + +enum udma_rx_limit_wl { + UDMA_RX_LIMIT_WL_0 = 0, + UDMA_RX_LIMIT_WL_64, + UDMA_RX_LIMIT_WL_512, + UDMA_RX_LIMIT_WL_4096 +}; + +enum { + LIMIT_WL_0_V = 0, + LIMIT_WL_64_V = 64, + LIMIT_WL_512_V = 512, + LIMIT_WL_4096_V = 4096 +}; + +struct udma_jfr_idx_que { + struct udma_buf buf; + struct udma_table jfr_idx_table; +}; + +struct udma_jfr { + struct ubcore_jfr ubcore_jfr; + struct udma_jetty_queue rq; + struct udma_jfr_idx_que idx_que; + struct udma_sw_db sw_db; + struct udma_sw_db jfr_sleep_buf; + struct udma_context *udma_ctx; + uint32_t rx_threshold; + uint32_t wqe_cnt; + uint64_t jetty_addr; + enum ubcore_jfr_state state; + uint32_t max_sge; + spinlock_t lock; + refcount_t ae_refcount; + struct completion ae_comp; +}; + +struct udma_wqe_sge { + uint32_t length; + uint32_t token_id; + uint64_t va; +}; + +struct udma_jfr_ctx { + /* DW0 */ + uint32_t state : 2; + uint32_t limit_wl : 2; + uint32_t rqe_size_shift : 3; + uint32_t token_en : 1; + uint32_t rqe_shift : 4; + uint32_t rnr_timer : 5; + uint32_t record_db_en : 1; + uint32_t rqe_token_id_l : 14; + /* DW1 */ + uint32_t rqe_token_id_h : 6; + uint32_t type : 3; + uint32_t rsv : 3; + uint32_t rqe_base_addr_l : 20; + /* DW2 */ + uint32_t rqe_base_addr_h; + /* DW3 */ + uint32_t rqe_position : 1; + uint32_t pld_position : 1; + uint32_t pld_token_id : 20; + uint32_t rsv1 : 10; + /* DW4 */ + uint32_t token_value; + /* DW5 */ + uint32_t user_data_l; + /* DW6 */ + uint32_t user_data_h; + /* DW7 */ + uint32_t pi : 16; + uint32_t ci : 16; + /* DW8 */ + uint32_t idx_que_addr_l; + /* DW9 */ + uint32_t idx_que_addr_h : 20; + uint32_t jfcn_l : 12; + /* DW10 */ + uint32_t jfcn_h : 8; + uint32_t record_db_addr_l : 24; + /* DW11 */ + uint32_t record_db_addr_m; + /* DW12 */ + uint32_t record_db_addr_h : 2; + uint32_t cqeie : 1; + uint32_t cqesz : 1; + uint32_t rsv2 : 28; + /* padding */ + uint32_t reserved[3]; +}; + +static inline struct udma_jfr *to_udma_jfr(struct ubcore_jfr *jfr) +{ + return container_of(jfr, struct udma_jfr, ubcore_jfr); +} + +static inline bool udma_jfrwq_overflow(struct udma_jfr *jfr) +{ + return (jfr->rq.pi - jfr->rq.ci) >= jfr->wqe_cnt; +} + +static inline void set_data_of_sge(struct udma_wqe_sge *sge, struct ubcore_sge *sg) +{ + sge->va = cpu_to_le64(sg->addr); + sge->length = cpu_to_le32(sg->len); +} + +static inline struct udma_jfr *to_udma_jfr_from_queue(struct udma_jetty_queue *queue) +{ + return container_of(queue, struct udma_jfr, rq); +} + +int udma_modify_jfr(struct ubcore_jfr *jfr, struct ubcore_jfr_attr *attr, + struct ubcore_udata *udata); +struct ubcore_jfr *udma_create_jfr(struct ubcore_device *dev, struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jfr(struct ubcore_jfr *jfr); +int udma_destroy_jfr_batch(struct ubcore_jfr **jfr_arr, int jfr_num, int *bad_jfr_index); +int udma_unimport_jfr(struct ubcore_tjetty *tjfr); +struct ubcore_tjetty *udma_import_jfr_ex(struct ubcore_device *dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_active_tp_cfg *active_tp_cfg, + struct ubcore_udata *udata); +int udma_post_jfr_wr(struct ubcore_jfr *ubcore_jfr, struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr); + +#endif /* __UDMA_JFR_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_jfs.c b/drivers/ub/urma/hw/udma/udma_jfs.c new file mode 100644 index 0000000000000000000000000000000000000000..5d520a0cea00bfb2bf3e9abf3e6d2fe6ef4fe23e --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_jfs.c @@ -0,0 +1,1287 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt +#define pr_fmt(fmt) "UDMA: " fmt + +#include +#include +#include +#include +#include +#include +#include "udma_common.h" +#include "udma_dev.h" +#include +#include "udma_cmd.h" +#include "udma_jetty.h" +#include "udma_segment.h" +#include "udma_jfs.h" + +int udma_alloc_u_sq_buf(struct udma_dev *dev, struct udma_jetty_queue *sq, + struct udma_create_jetty_ucmd *ucmd) +{ + int ret = 0; + + if (ucmd->sqe_bb_cnt == 0 || ucmd->buf_len == 0) { + dev_err(dev->dev, "invalid param, sqe_bb_cnt=%u, buf_len=%u.\n", + ucmd->sqe_bb_cnt, ucmd->buf_len); + return -EINVAL; + } + + sq->sqe_bb_cnt = ucmd->sqe_bb_cnt; + sq->buf.entry_cnt = ucmd->buf_len >> WQE_BB_SIZE_SHIFT; + if (sq->non_pin) { + sq->buf.addr = ucmd->buf_addr; + } else if (ucmd->is_hugepage) { + sq->buf.addr = ucmd->buf_addr; + if (udma_occupy_u_hugepage(sq->udma_ctx, (void *)sq->buf.addr)) { + dev_err(dev->dev, "failed to create sq, va not map.\n"); + return -EINVAL; + } + sq->buf.is_hugepage = true; + } else { + ret = pin_queue_addr(dev, ucmd->buf_addr, ucmd->buf_len, &sq->buf); + if (ret) { + dev_err(dev->dev, "failed to pin sq, ret = %d.\n", ret); + return ret; + } + } + + return ret; +} + +int udma_alloc_k_sq_buf(struct udma_dev *dev, struct udma_jetty_queue *sq, + struct ubcore_jfs_cfg *jfs_cfg) +{ + uint32_t wqe_bb_depth; + uint32_t sqe_bb_cnt; + uint32_t size; + int ret; + + if (!jfs_cfg->flag.bs.lock_free) + spin_lock_init(&sq->lock); + + sq->max_inline_size = jfs_cfg->max_inline_data; + sq->max_sge_num = jfs_cfg->max_sge; + sq->tid = dev->tid; + sq->lock_free = jfs_cfg->flag.bs.lock_free; + + sqe_bb_cnt = sq_cal_wqebb_num(SQE_WRITE_NOTIFY_CTL_LEN, jfs_cfg->max_sge); + sq->sqe_bb_cnt = sqe_bb_cnt > (uint32_t)MAX_WQEBB_NUM ? (uint32_t)MAX_WQEBB_NUM : + sqe_bb_cnt; + + wqe_bb_depth = roundup_pow_of_two(sq->sqe_bb_cnt * jfs_cfg->depth); + sq->buf.entry_size = UDMA_JFS_WQEBB_SIZE; + size = ALIGN(wqe_bb_depth * sq->buf.entry_size, UDMA_HW_PAGE_SIZE); + sq->buf.entry_cnt = size >> WQE_BB_SIZE_SHIFT; + + ret = udma_k_alloc_buf(dev, &sq->buf); + if (ret) { + dev_err(dev->dev, "failed to alloc sq buffer, id=%u.\n", sq->id); + return ret; + } + + sq->wrid = kcalloc(1, sq->buf.entry_cnt * sizeof(uint64_t), GFP_KERNEL); + if (!sq->wrid) { + dev_err(dev->dev, + "failed to alloc wrid for jfs id = %u when entry cnt = %u.\n", + sq->id, sq->buf.entry_cnt); + udma_k_free_buf(dev, &sq->buf); + return -ENOMEM; + } + + udma_alloc_kernel_db(dev, sq); + sq->kva_curr = sq->buf.kva; + + return 0; +} + +void udma_free_sq_buf(struct udma_dev *dev, struct udma_jetty_queue *sq) +{ + if (sq->buf.kva) { + udma_k_free_buf(dev, &sq->buf); + kfree(sq->wrid); + return; + } + + if (sq->non_pin) + return; + + if (sq->buf.is_hugepage) { + udma_return_u_hugepage(sq->udma_ctx, (void *)sq->buf.addr); + } else { + unpin_queue_addr(sq->buf.umem); + } +} + +void udma_init_jfsc(struct udma_dev *dev, struct ubcore_jfs_cfg *cfg, + struct udma_jfs *jfs, void *mb_buf) +{ + struct udma_jetty_ctx *ctx = (struct udma_jetty_ctx *)mb_buf; + uint8_t i; + + ctx->state = JETTY_READY; + ctx->jfs_mode = JFS; + ctx->type = to_udma_type(cfg->trans_mode); + ctx->sl = dev->udma_sl[UDMA_DEFAULT_SL_NUM]; + if (ctx->type == JETTY_RM || ctx->type == JETTY_RC) { + for (i = 0; i < dev->udma_total_sl_num; i++) + if (cfg->priority == dev->udma_sl[i]) + ctx->sl = cfg->priority; + } else if (ctx->type == JETTY_UM) { + ctx->sl = dev->unic_sl[UDMA_DEFAULT_SL_NUM]; + for (i = 0; i < dev->unic_sl_num; i++) + if (cfg->priority == dev->unic_sl[i]) + ctx->sl = cfg->priority; + } + ctx->sqe_base_addr_l = (jfs->sq.buf.addr >> SQE_VA_L_OFFSET) & + (uint32_t)SQE_VA_L_VALID_BIT; + ctx->sqe_base_addr_h = (jfs->sq.buf.addr >> SQE_VA_H_OFFSET) & + (uint32_t)SQE_VA_H_VALID_BIT; + ctx->sqe_token_id_l = jfs->sq.tid & (uint32_t)SQE_TOKEN_ID_L_MASK; + ctx->sqe_token_id_h = (jfs->sq.tid >> SQE_TOKEN_ID_H_OFFSET) & + (uint32_t)SQE_TOKEN_ID_H_MASK; + ctx->sqe_bb_shift = ilog2(roundup_pow_of_two(jfs->sq.buf.entry_cnt)); + ctx->tx_jfcn = cfg->jfc->id; + ctx->ta_timeout = to_ta_timeout(cfg->err_timeout); + + if (!!(dev->caps.feature & UDMA_CAP_FEATURE_RNR_RETRY)) + ctx->rnr_retry_num = cfg->rnr_retry; + + ctx->user_data_l = jfs->jfs_addr; + ctx->user_data_h = jfs->jfs_addr >> UDMA_USER_DATA_H_OFFSET; + ctx->seid_idx = cfg->eid_index; + ctx->err_mode = cfg->flag.bs.error_suspend; + ctx->cmp_odr = cfg->flag.bs.outorder_comp; + ctx->avail_sgmt_ost = AVAIL_SGMT_OST_INIT; + ctx->pi_type = jfs->pi_type; + ctx->sqe_pld_tokenid = jfs->sq.tid & (uint32_t)SQE_PLD_TOKEN_ID_MASK; + ctx->next_send_ssn = get_random_u16(); + ctx->next_rcv_ssn = ctx->next_send_ssn; +} + +int udma_verify_jfs_param(struct udma_dev *dev, struct ubcore_jfs_cfg *cfg, + bool enable_stars) +{ + if (!cfg->depth || cfg->depth > dev->caps.jfs.depth || + cfg->max_sge > dev->caps.jfs_sge || cfg->trans_mode == UBCORE_TP_RC) { + dev_err(dev->dev, + "jfs param is invalid, depth = %u, seg = %u, max_depth = %u, max_jfs_seg = %u, trans_mode = %u.\n", + cfg->depth, cfg->max_sge, dev->caps.jfs.depth, + dev->caps.jfs_sge, cfg->trans_mode); + return -EINVAL; + } + + if (enable_stars && cfg->max_inline_data != 0 && + cfg->max_inline_data > dev->caps.jfs_inline_sz) { + dev_err(dev->dev, "jfs param is invalid, inline_data:%u, max_inline_len:%u.\n", + cfg->max_inline_data, dev->caps.jfs_inline_sz); + return -EINVAL; + } + + if (enable_stars && cfg->max_rsge > dev->caps.jfs_rsge) { + dev_err(dev->dev, "jfs param is invalid, rsge:%u, max_rsge:%u.\n", + cfg->max_rsge, dev->caps.jfs_rsge); + return -EINVAL; + } + + if (cfg->priority >= UDMA_MAX_PRIORITY) { + dev_err(dev->dev, "kernel mode jfs priority is out of range, priority is %u.\n", + cfg->priority); + return -EINVAL; + } + + return 0; +} + +void udma_dfx_store_jfs_id(struct udma_dev *udma_dev, struct udma_jfs *udma_jfs) +{ + struct udma_dfx_jfs *jfs; + int ret; + + jfs = (struct udma_dfx_jfs *)xa_load(&udma_dev->dfx_info->jfs.table, + udma_jfs->sq.id); + if (jfs) { + dev_warn(udma_dev->dev, "jfs_id(%u) already exists in DFX.\n", + udma_jfs->sq.id); + return; + } + + jfs = kzalloc(sizeof(*jfs), GFP_KERNEL); + if (!jfs) + return; + + jfs->id = udma_jfs->sq.id; + jfs->depth = udma_jfs->sq.buf.entry_cnt / udma_jfs->sq.sqe_bb_cnt; + + write_lock(&udma_dev->dfx_info->jfs.rwlock); + ret = xa_err(xa_store(&udma_dev->dfx_info->jfs.table, udma_jfs->sq.id, + jfs, GFP_KERNEL)); + if (ret) { + write_unlock(&udma_dev->dfx_info->jfs.rwlock); + dev_err(udma_dev->dev, "store jfs_id(%u) to table failed in DFX.\n", + udma_jfs->sq.id); + kfree(jfs); + return; + } + + ++udma_dev->dfx_info->jfs.cnt; + write_unlock(&udma_dev->dfx_info->jfs.rwlock); +} + +static int udma_create_hw_jfs_ctx(struct udma_dev *dev, struct udma_jfs *jfs, + struct ubcore_jfs_cfg *cfg) +{ + struct ubase_mbx_attr attr = {}; + struct udma_jetty_ctx ctx = {}; + int ret; + + if (cfg->priority >= UDMA_MAX_PRIORITY) { + dev_err(dev->dev, "kernel mode jfs priority is out of range, priority is %u.\n", + cfg->priority); + return -EINVAL; + } + + udma_init_jfsc(dev, cfg, jfs, &ctx); + attr.tag = jfs->sq.id; + attr.op = UDMA_CMD_CREATE_JFS_CONTEXT; + ret = post_mailbox_update_ctx(dev, &ctx, sizeof(ctx), &attr); + if (ret) { + dev_err(dev->dev, "failed to upgrade JFSC, ret = %d.\n", ret); + return ret; + } + + return 0; +} + +static int udma_get_user_jfs_cmd(struct udma_dev *dev, struct udma_jfs *jfs, + struct ubcore_udata *udata, + struct udma_create_jetty_ucmd *ucmd) +{ + struct udma_context *uctx; + unsigned long byte; + + if (udata) { + if (!udata->udrv_data) { + dev_err(dev->dev, "udrv_data is null.\n"); + return -EINVAL; + } + + if (!udata->udrv_data->in_addr || udata->udrv_data->in_len < sizeof(*ucmd)) { + dev_err(dev->dev, "jfs in_len %u or addr is invalid.\n", + udata->udrv_data->in_len); + return -EINVAL; + } + + byte = copy_from_user(ucmd, (void *)(uintptr_t)udata->udrv_data->in_addr, + sizeof(*ucmd)); + if (byte) { + dev_err(dev->dev, + "failed to copy jfs udata, ret = %lu.\n", byte); + return -EFAULT; + } + + uctx = to_udma_context(udata->uctx); + jfs->sq.udma_ctx = uctx; + jfs->sq.tid = uctx->tid; + jfs->jfs_addr = ucmd->jetty_addr; + jfs->pi_type = ucmd->pi_type; + jfs->sq.non_pin = ucmd->non_pin; + jfs->sq.jetty_type = (enum udma_jetty_type)ucmd->jetty_type; + jfs->sq.id = ucmd->jfs_id; + } else { + jfs->jfs_addr = (uintptr_t)&jfs->sq; + jfs->sq.jetty_type = (enum udma_jetty_type)UDMA_URMA_NORMAL_JETTY_TYPE; + } + + return 0; +} + +static int udma_alloc_jfs_sq(struct udma_dev *dev, struct ubcore_jfs_cfg *cfg, + struct udma_jfs *jfs, struct ubcore_udata *udata) +{ + struct udma_create_jetty_ucmd ucmd = {}; + int ret; + + ret = udma_get_user_jfs_cmd(dev, jfs, udata, &ucmd); + if (ret) + goto err_get_user_cmd; + + ret = alloc_jetty_id(dev, &jfs->sq, jfs->sq.id, NULL); + if (ret) { + dev_err(dev->dev, "failed to alloc_id.\n"); + goto err_alloc_id; + } + jfs->ubcore_jfs.jfs_id.id = jfs->sq.id; + jfs->ubcore_jfs.jfs_cfg = *cfg; + udma_set_query_flush_time(&jfs->sq, cfg->err_timeout); + + ret = xa_err(xa_store(&dev->jetty_table.xa, jfs->sq.id, &jfs->sq, GFP_KERNEL)); + if (ret) { + dev_err(dev->dev, "failed to store_sq(%u), ret=%d.", jfs->sq.id, ret); + goto err_store_sq; + } + + ret = udata ? udma_alloc_u_sq_buf(dev, &jfs->sq, &ucmd) : + udma_alloc_k_sq_buf(dev, &jfs->sq, cfg); + if (ret) + goto err_alloc_sq_buf; + + jfs->sq.trans_mode = cfg->trans_mode; + + return ret; + +err_alloc_sq_buf: + xa_erase(&dev->jetty_table.xa, jfs->sq.id); +err_store_sq: + if (jfs->sq.id < dev->caps.jetty.start_idx) + udma_id_free(&dev->rsvd_jetty_ida_table, jfs->sq.id); + else + udma_adv_id_free(&dev->jetty_table.bitmap_table, + jfs->sq.id, false); +err_alloc_id: +err_get_user_cmd: + return ret; +} + +struct ubcore_jfs *udma_create_jfs(struct ubcore_device *ub_dev, + struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *dev = to_udma_dev(ub_dev); + struct udma_jfs *jfs; + int ret; + + if (cfg->trans_mode == UBCORE_TP_RC) { + dev_err(dev->dev, "jfs not support RC transmode.\n"); + return NULL; + } + + jfs = kcalloc(1, sizeof(*jfs), GFP_KERNEL); + if (!jfs) + return NULL; + + ret = udma_alloc_jfs_sq(dev, cfg, jfs, udata); + if (ret) { + dev_err(dev->dev, "failed to alloc_jfs_sq, ret = %d.\n", ret); + goto err_alloc_sq; + } + + ret = udma_create_hw_jfs_ctx(dev, jfs, cfg); + if (ret) { + dev_err(dev->dev, + "post mailbox create jfs ctx failed, ret = %d.\n", ret); + goto err_create_hw_jfs; + } + + jfs->mode = UDMA_NORMAL_JFS_TYPE; + jfs->sq.state = UBCORE_JETTY_STATE_READY; + refcount_set(&jfs->ae_refcount, 1); + init_completion(&jfs->ae_comp); + if (dfx_switch) + udma_dfx_store_jfs_id(dev, jfs); + + return &jfs->ubcore_jfs; + +err_create_hw_jfs: + udma_free_sq_buf(dev, &jfs->sq); + xa_erase(&dev->jetty_table.xa, jfs->sq.id); + if (jfs->sq.id < dev->caps.jetty.start_idx) + udma_id_free(&dev->rsvd_jetty_ida_table, jfs->sq.id); + else + udma_adv_id_free(&dev->jetty_table.bitmap_table, + jfs->sq.id, false); +err_alloc_sq: + kfree(jfs); + return NULL; +} + +static void udma_free_jfs(struct ubcore_jfs *jfs) +{ + struct udma_dev *dev = to_udma_dev(jfs->ub_dev); + struct udma_jfs *ujfs = to_udma_jfs(jfs); + + udma_clean_cqe_for_jetty(dev, &ujfs->sq, jfs->jfs_cfg.jfc, NULL); + + xa_erase(&dev->jetty_table.xa, ujfs->sq.id); + + if (refcount_dec_and_test(&ujfs->ae_refcount)) + complete(&ujfs->ae_comp); + wait_for_completion(&ujfs->ae_comp); + + if (dfx_switch) + udma_dfx_delete_id(dev, &dev->dfx_info->jfs, jfs->jfs_id.id); + + if (ujfs->mode == UDMA_NORMAL_JFS_TYPE) + udma_free_sq_buf(dev, &ujfs->sq); + else + kfree(ujfs->sq.wrid); + + if (ujfs->sq.id < dev->caps.jetty.start_idx) + udma_id_free(&dev->rsvd_jetty_ida_table, ujfs->sq.id); + else + udma_adv_id_free(&dev->jetty_table.bitmap_table, + ujfs->sq.id, false); + + kfree(ujfs); +} + +int udma_destroy_jfs(struct ubcore_jfs *jfs) +{ + struct udma_dev *dev = to_udma_dev(jfs->ub_dev); + struct udma_jfs *ujfs = to_udma_jfs(jfs); + int ret; + + if (!ujfs->ue_rx_closed && udma_close_ue_rx(dev, true, true, false, 0)) { + dev_err(dev->dev, "close ue rx failed when destroying jfs.\n"); + return -EINVAL; + } + + ret = udma_modify_and_destroy_jetty(dev, &ujfs->sq); + if (ret) { + dev_info(dev->dev, "udma modify error and destroy jfs failed, id: %u.\n", + jfs->jfs_id.id); + if (!ujfs->ue_rx_closed) + udma_open_ue_rx(dev, true, true, false, 0); + return ret; + } + + udma_free_jfs(jfs); + udma_open_ue_rx(dev, true, true, false, 0); + + return 0; +} + +int udma_destroy_jfs_batch(struct ubcore_jfs **jfs, int jfs_cnt, int *bad_jfs_index) +{ + struct udma_jetty_queue **sq_list; + struct udma_dev *udma_dev; + uint32_t i; + int ret; + + if (!jfs) { + pr_err("jfs array is null.\n"); + return -EINVAL; + } + + if (!jfs_cnt) { + pr_err("jfs cnt is 0.\n"); + return -EINVAL; + } + + udma_dev = to_udma_dev(jfs[0]->ub_dev); + + sq_list = kcalloc(jfs_cnt, sizeof(*sq_list), GFP_KERNEL); + if (!sq_list) + return -ENOMEM; + + for (i = 0; i < jfs_cnt; i++) + sq_list[i] = &(to_udma_jfs(jfs[i])->sq); + + ret = udma_batch_modify_and_destroy_jetty(udma_dev, sq_list, jfs_cnt, bad_jfs_index); + + kfree(sq_list); + + if (ret) { + dev_err(udma_dev->dev, + "udma batch modify error and destroy jfs failed.\n"); + return ret; + } + + for (i = 0; i < jfs_cnt; i++) + udma_free_jfs(jfs[i]); + + return 0; +} + +static int udma_modify_jfs_state(struct udma_dev *udma_dev, struct udma_jfs *udma_jfs, + struct ubcore_jfs_attr *attr) +{ + int ret; + + switch (attr->state) { + case UBCORE_JETTY_STATE_RESET: + ret = udma_destroy_hw_jetty_ctx(udma_dev, udma_jfs->sq.id); + break; + case UBCORE_JETTY_STATE_READY: + ret = udma_create_hw_jfs_ctx(udma_dev, udma_jfs, &udma_jfs->ubcore_jfs.jfs_cfg); + if (ret) + break; + + udma_reset_sw_k_jetty_queue(&udma_jfs->sq); + break; + default: + ret = udma_close_ue_rx(udma_dev, true, true, false, 0); + if (ret) + break; + + if (!(udma_dev->caps.feature & UDMA_CAP_FEATURE_UE_RX_CLOSE)) { + if (udma_modify_jetty_precondition(udma_dev, &udma_jfs->sq)) { + ret = -ENOMEM; + udma_open_ue_rx(udma_dev, true, true, false, 0); + break; + } + } + + ret = udma_set_jetty_state(udma_dev, udma_jfs->sq.id, to_jetty_state(attr->state)); + if (ret) + udma_open_ue_rx(udma_dev, true, true, false, 0); + else + udma_jfs->ue_rx_closed = true; + break; + } + + return ret; +} + +int udma_modify_jfs(struct ubcore_jfs *jfs, struct ubcore_jfs_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(jfs->ub_dev); + struct udma_jfs *udma_jfs = to_udma_jfs(jfs); + int ret = 0; + + if (!(attr->mask & UBCORE_JFS_STATE)) { + dev_err(udma_dev->dev, "modify jfs mask is error or not set, jfs_id = %u.\n", + udma_jfs->sq.id); + return -EINVAL; + } + + if (udma_jfs->sq.state == attr->state) { + dev_info(udma_dev->dev, "jfs state has been %s.\n", + to_state_name(attr->state)); + return 0; + } + + if (!verify_modify_jetty(udma_jfs->sq.state, attr->state)) { + dev_err(udma_dev->dev, "not support modify jfs state from %s to %s.\n", + to_state_name(udma_jfs->sq.state), to_state_name(attr->state)); + return -EINVAL; + } + + ret = udma_modify_jfs_state(udma_dev, udma_jfs, attr); + if (ret) { + dev_err(udma_dev->dev, "modify jfs %u state to %u failed.\n", + udma_jfs->sq.id, attr->state); + return ret; + } + + udma_jfs->sq.state = attr->state; + + return 0; +} + +static void fill_imm_data_or_token_for_cr(struct udma_dev *udma_dev, + struct udma_sqe_ctl *sqe_ctl, + struct ubcore_cr *cr, + uint32_t opcode) +{ + switch (opcode) { + case UDMA_OPC_SEND: + case UDMA_OPC_WRITE: + case UDMA_OPC_READ: + case UDMA_OPC_CAS: + case UDMA_OPC_FAA: + break; + case UDMA_OPC_SEND_WITH_IMM: + memcpy(&cr->imm_data, (void *)sqe_ctl + SQE_SEND_IMM_FIELD, + sizeof(uint64_t)); + break; + case UDMA_OPC_SEND_WITH_INVALID: + cr->invalid_token.token_id = sqe_ctl->rmt_addr_l_or_token_id; + cr->invalid_token.token_value.token = sqe_ctl->rmt_addr_h_or_token_value; + break; + case UDMA_OPC_WRITE_WITH_IMM: + memcpy(&cr->imm_data, (void *)sqe_ctl + SQE_WRITE_IMM_FIELD, + sizeof(uint64_t)); + break; + default: + dev_err(udma_dev->dev, "Flush invalid opcode :%u.\n", opcode); + break; + } +} + +static void fill_cr_by_sqe_ctl(struct udma_dev *udma_dev, + struct udma_sqe_ctl *sqe_ctl, + struct ubcore_cr *cr) +{ + uint32_t opcode = sqe_ctl->opcode; + struct udma_normal_sge *sge; + uint32_t src_sge_num = 0; + uint64_t total_len = 0; + uint32_t ctrl_len; + uint32_t i; + + fill_imm_data_or_token_for_cr(udma_dev, sqe_ctl, cr, opcode); + + cr->tpn = sqe_ctl->tpn; + cr->remote_id.id = sqe_ctl->rmt_obj_id; + memcpy(cr->remote_id.eid.raw, sqe_ctl->rmt_eid, UBCORE_EID_SIZE); + + if (sqe_ctl->inline_en) { + cr->completion_len = sqe_ctl->inline_msg_len; + return; + } + + src_sge_num = sqe_ctl->sge_num; + ctrl_len = get_ctl_len(opcode); + sge = (struct udma_normal_sge *)((void *)sqe_ctl + ctrl_len); + + for (i = 0; i < src_sge_num; i++) { + total_len += sge->length; + sge++; + } + + if (total_len > UINT32_MAX) { + cr->completion_len = UINT32_MAX; + dev_warn(udma_dev->dev, "total len %llu is overflow.\n", total_len); + } else { + cr->completion_len = total_len; + } +} + +static void udma_copy_from_sq(struct udma_jetty_queue *sq, uint32_t wqebb_cnt, + struct udma_jfs_wqebb *tmp_sq) +{ + uint32_t field_h; + uint32_t field_l; + uint32_t offset; + uint32_t remain; + + remain = sq->buf.entry_cnt - (sq->ci & (sq->buf.entry_cnt - 1)); + offset = (sq->ci & (sq->buf.entry_cnt - 1)) * UDMA_JFS_WQEBB_SIZE; + field_h = remain > wqebb_cnt ? wqebb_cnt : remain; + field_l = wqebb_cnt > field_h ? wqebb_cnt - field_h : 0; + + memcpy(tmp_sq, sq->buf.kva + offset, field_h * sizeof(*tmp_sq)); + + if (field_l) + memcpy(tmp_sq + field_h, sq->buf.kva, field_l * sizeof(*tmp_sq)); +} + +static uint32_t get_wqebb_num(struct udma_sqe_ctl *sqe_ctl) +{ + uint32_t opcode = sqe_ctl->opcode; + uint32_t sqe_ctl_len = get_ctl_len(opcode); + + switch (opcode) { + case UDMA_OPC_SEND: + case UDMA_OPC_SEND_WITH_IMM: + case UDMA_OPC_SEND_WITH_INVALID: + case UDMA_OPC_WRITE: + case UDMA_OPC_WRITE_WITH_IMM: + if (sqe_ctl->inline_en) + return (sqe_ctl_len + sqe_ctl->inline_msg_len - 1) / + UDMA_JFS_WQEBB_SIZE + 1; + break; + case UDMA_OPC_CAS: + case UDMA_OPC_FAA: + return ATOMIC_WQEBB_CNT; + case UDMA_OPC_NOP: + return NOP_WQEBB_CNT; + default: + break; + } + + return sq_cal_wqebb_num(sqe_ctl_len, sqe_ctl->sge_num); +} + +void udma_flush_sq(struct udma_dev *udma_dev, + struct udma_jetty_queue *sq, struct ubcore_cr *cr) +{ + struct udma_jfs_wqebb tmp_sq[MAX_WQEBB_NUM] = {}; + + udma_copy_from_sq(sq, MAX_WQEBB_NUM, tmp_sq); + fill_cr_by_sqe_ctl(udma_dev, (struct udma_sqe_ctl *)tmp_sq, cr); + cr->status = UBCORE_CR_WR_UNHANDLED; + cr->user_ctx = sq->wrid[sq->ci & (sq->buf.entry_cnt - 1)]; + /* Fill in UINT8_MAX for send direction */ + cr->opcode = UINT8_MAX; + cr->local_id = sq->id; + + sq->ci += get_wqebb_num((struct udma_sqe_ctl *)tmp_sq); +} + +int udma_flush_jfs(struct ubcore_jfs *jfs, int cr_cnt, struct ubcore_cr *cr) +{ + struct udma_dev *udma_dev = to_udma_dev(jfs->ub_dev); + struct udma_jfs *udma_jfs = to_udma_jfs(jfs); + struct udma_jetty_queue *sq = &udma_jfs->sq; + int n_flushed; + + if (!sq->flush_flag) + return 0; + + if (!jfs->jfs_cfg.flag.bs.lock_free) + spin_lock(&sq->lock); + + for (n_flushed = 0; n_flushed < cr_cnt; n_flushed++) { + if (sq->ci == sq->pi) + break; + udma_flush_sq(udma_dev, sq, cr + n_flushed); + } + + if (!jfs->jfs_cfg.flag.bs.lock_free) + spin_unlock(&sq->lock); + + return n_flushed; +} + +static uint8_t udma_get_jfs_opcode(enum ubcore_opcode opcode) +{ + switch (opcode) { + case UBCORE_OPC_SEND: + return UDMA_OPC_SEND; + case UBCORE_OPC_SEND_IMM: + return UDMA_OPC_SEND_WITH_IMM; + case UBCORE_OPC_SEND_INVALIDATE: + return UDMA_OPC_SEND_WITH_INVALID; + case UBCORE_OPC_WRITE: + return UDMA_OPC_WRITE; + case UBCORE_OPC_WRITE_IMM: + return UDMA_OPC_WRITE_WITH_IMM; + case UBCORE_OPC_READ: + return UDMA_OPC_READ; + case UBCORE_OPC_CAS: + return UDMA_OPC_CAS; + case UBCORE_OPC_FADD: + return UDMA_OPC_FAA; + case UBCORE_OPC_NOP: + return UDMA_OPC_NOP; + default: + return UDMA_OPC_INVALID; + } +} + +static int +udma_fill_sw_sge(struct udma_dev *dev, struct udma_sqe_ctl *sqe_ctl, + struct ubcore_jfs_wr *wr, uint32_t max_inline_size, + struct udma_normal_sge *sge) +{ + struct ubcore_sge *sge_info; + uint32_t total_len = 0; + uint32_t sge_num = 0; + uint32_t num_sge; + uint32_t i; + + switch (wr->opcode) { + case UBCORE_OPC_SEND: + case UBCORE_OPC_SEND_IMM: + case UBCORE_OPC_SEND_INVALIDATE: + sge_info = wr->send.src.sge; + num_sge = wr->send.src.num_sge; + break; + case UBCORE_OPC_WRITE: + case UBCORE_OPC_WRITE_IMM: + sge_info = wr->rw.src.sge; + num_sge = wr->rw.src.num_sge; + break; + default: + return -EINVAL; + } + + if (wr->flag.bs.inline_flag) { + for (i = 0; i < num_sge; i++) { + if (total_len + sge_info[i].len > max_inline_size) { + dev_info(dev->dev, "inline_size %u is over max_size %u.\n", + total_len + sge_info[i].len, max_inline_size); + return -EINVAL; + } + + memcpy((void *)(uintptr_t)sge + total_len, + (void *)(uintptr_t)sge_info[i].addr, + sge_info[i].len); + total_len += sge_info[i].len; + } + sqe_ctl->inline_msg_len = total_len; + } else { + for (i = 0; i < num_sge; i++) { + if (sge_info[i].len == 0) + continue; + sge->va = sge_info[i].addr; + sge->length = sge_info[i].len; + sge++; + sge_num++; + } + sqe_ctl->sge_num = sge_num; + } + + return 0; +} + +static int +udma_k_fill_send_sqe(struct udma_dev *dev, struct udma_sqe_ctl *sqe_ctl, + struct ubcore_jfs_wr *wr, struct ubcore_tjetty *tjetty, + uint32_t max_inline_size) +{ + struct udma_target_jetty *udma_tjetty; + struct udma_token_info *token_info; + struct udma_segment *udma_seg; + struct udma_normal_sge *sge; + + sge = (struct udma_normal_sge *)(sqe_ctl + 1); + + if (udma_fill_sw_sge(dev, sqe_ctl, wr, max_inline_size, sge)) + return -EINVAL; + + udma_tjetty = to_udma_tjetty(tjetty); + sqe_ctl->target_hint = wr->send.target_hint; + sqe_ctl->rmt_obj_id = tjetty->cfg.id.id; + sqe_ctl->token_en = udma_tjetty->token_value_valid; + sqe_ctl->rmt_token_value = udma_tjetty->token_value; + + if (wr->opcode == UBCORE_OPC_SEND_IMM) { + memcpy((void *)sqe_ctl + SQE_SEND_IMM_FIELD, &wr->send.imm_data, + sizeof(uint64_t)); + } else if (wr->opcode == UBCORE_OPC_SEND_INVALIDATE) { + udma_seg = to_udma_seg(wr->send.tseg); + token_info = (struct udma_token_info *)&sqe_ctl->rmt_addr_l_or_token_id; + token_info->token_id = udma_seg->tid; + token_info->token_value = udma_seg->token_value; + } + + return 0; +} + +static int +udma_k_fill_write_sqe(struct udma_dev *dev, struct udma_sqe_ctl *sqe_ctl, + struct ubcore_jfs_wr *wr, struct ubcore_tjetty *tjetty, + uint32_t max_inline_size) +{ + struct udma_token_info *token_info; + struct udma_segment *udma_seg; + struct udma_normal_sge *sge; + struct ubcore_sge *sge_info; + uint32_t ctrl_len; + + ctrl_len = get_ctl_len(sqe_ctl->opcode); + sge = (struct udma_normal_sge *)((void *)sqe_ctl + ctrl_len); + + if (udma_fill_sw_sge(dev, sqe_ctl, wr, max_inline_size, sge)) + return -EINVAL; + + sge_info = wr->rw.dst.sge; + udma_seg = to_udma_seg(sge_info[0].tseg); + + sqe_ctl->target_hint = wr->rw.target_hint; + sqe_ctl->rmt_obj_id = udma_seg->tid; + sqe_ctl->token_en = udma_seg->token_value_valid; + sqe_ctl->rmt_token_value = udma_seg->token_value; + sqe_ctl->rmt_addr_l_or_token_id = sge_info[0].addr & (uint32_t)SQE_CTL_RMA_ADDR_BIT; + sqe_ctl->rmt_addr_h_or_token_value = + (sge_info[0].addr >> (uint32_t)SQE_CTL_RMA_ADDR_OFFSET) & + (uint32_t)SQE_CTL_RMA_ADDR_BIT; + + if (sqe_ctl->opcode == UDMA_OPC_WRITE_WITH_IMM) { + memcpy((void *)sqe_ctl + SQE_WRITE_IMM_FIELD, &wr->rw.notify_data, + sizeof(uint64_t)); + token_info = (struct udma_token_info *) + ((void *)sqe_ctl + WRITE_IMM_TOKEN_FIELD); + token_info->token_id = tjetty->cfg.id.id; + token_info->token_value = tjetty->cfg.token_value.token; + } + + return 0; +} + +static int udma_k_fill_read_sqe(struct udma_sqe_ctl *sqe_ctl, struct ubcore_jfs_wr *wr) +{ + struct udma_segment *udma_seg; + struct udma_normal_sge *sge; + struct ubcore_sge *sge_info; + uint32_t sge_num = 0; + uint32_t num; + + sge = (struct udma_normal_sge *)(sqe_ctl + 1); + sge_info = wr->rw.dst.sge; + + for (num = 0; num < wr->rw.dst.num_sge; num++) { + if (sge_info[num].len == 0) + continue; + sge->va = sge_info[num].addr; + sge->length = sge_info[num].len; + sge++; + sge_num++; + } + + sge_info = wr->rw.src.sge; + udma_seg = to_udma_seg(sge_info[0].tseg); + + sqe_ctl->sge_num = sge_num; + sqe_ctl->rmt_obj_id = udma_seg->tid; + sqe_ctl->token_en = udma_seg->token_value_valid; + sqe_ctl->rmt_token_value = udma_seg->token_value; + sqe_ctl->rmt_addr_l_or_token_id = sge_info[0].addr & (uint32_t)SQE_CTL_RMA_ADDR_BIT; + sqe_ctl->rmt_addr_h_or_token_value = + (sge_info[0].addr >> (uint32_t)SQE_CTL_RMA_ADDR_OFFSET) & + (uint32_t)SQE_CTL_RMA_ADDR_BIT; + + return 0; +} + +static bool +udma_k_check_atomic_len(struct udma_dev *dev, uint32_t len, uint8_t opcode) +{ + switch (len) { + case UDMA_ATOMIC_LEN_4: + case UDMA_ATOMIC_LEN_8: + return true; + case UDMA_ATOMIC_LEN_16: + if (opcode == UBCORE_OPC_CAS) + return true; + dev_err(dev->dev, "the atomic opcode must be CAS when len is 16.\n"); + return false; + default: + dev_err(dev->dev, "invalid atomic len %u.\n", len); + return false; + } +} + +static int +udma_k_fill_cas_sqe(struct udma_dev *dev, struct udma_sqe_ctl *sqe_ctl, + struct ubcore_jfs_wr *wr) +{ + struct udma_segment *udma_seg; + struct udma_normal_sge *sge; + struct ubcore_sge *sge_info; + + sge_info = wr->cas.src; + if (!udma_k_check_atomic_len(dev, sge_info->len, wr->opcode)) + return -EINVAL; + + sge = (struct udma_normal_sge *)(sqe_ctl + 1); + + sge->va = sge_info->addr; + sge->length = sge_info->len; + + sge_info = wr->cas.dst; + udma_seg = to_udma_seg(sge_info->tseg); + + sqe_ctl->sge_num = UDMA_ATOMIC_SGE_NUM; + sqe_ctl->rmt_obj_id = udma_seg->tid; + sqe_ctl->token_en = udma_seg->token_value_valid; + sqe_ctl->rmt_token_value = udma_seg->token_value; + sqe_ctl->rmt_addr_l_or_token_id = sge_info->addr & (uint32_t)SQE_CTL_RMA_ADDR_BIT; + sqe_ctl->rmt_addr_h_or_token_value = (sge_info->addr >> (uint32_t)SQE_CTL_RMA_ADDR_OFFSET) & + (uint32_t)SQE_CTL_RMA_ADDR_BIT; + + if (sge->length <= UDMA_ATOMIC_LEN_8) { + memcpy((void *)sqe_ctl + SQE_ATOMIC_DATA_FIELD, + &wr->cas.swap_data, sge->length); + memcpy((void *)sqe_ctl + SQE_ATOMIC_DATA_FIELD + sge->length, + &wr->cas.cmp_data, sge->length); + } else { + memcpy((void *)sqe_ctl + SQE_ATOMIC_DATA_FIELD, + (void *)(uintptr_t)wr->cas.swap_addr, sge->length); + memcpy((void *)sqe_ctl + SQE_ATOMIC_DATA_FIELD + sge->length, + (void *)(uintptr_t)wr->cas.cmp_addr, sge->length); + } + + return 0; +} + +static int +udma_k_fill_faa_sqe(struct udma_dev *dev, struct udma_sqe_ctl *sqe_ctl, + struct ubcore_jfs_wr *wr) +{ + struct udma_segment *udma_seg; + struct udma_normal_sge *sge; + struct ubcore_sge *sge_info; + + sge_info = wr->faa.src; + if (!udma_k_check_atomic_len(dev, sge_info->len, wr->opcode)) + return -EINVAL; + + sge = (struct udma_normal_sge *)(sqe_ctl + 1); + + sge->va = sge_info->addr; + sge->length = sge_info->len; + + sge_info = wr->faa.dst; + udma_seg = to_udma_seg(sge_info->tseg); + + sqe_ctl->sge_num = UDMA_ATOMIC_SGE_NUM; + sqe_ctl->rmt_obj_id = udma_seg->tid; + sqe_ctl->token_en = udma_seg->token_value_valid; + sqe_ctl->rmt_token_value = udma_seg->token_value; + sqe_ctl->rmt_addr_l_or_token_id = sge_info->addr & (uint32_t)SQE_CTL_RMA_ADDR_BIT; + sqe_ctl->rmt_addr_h_or_token_value = (sge_info->addr >> (uint32_t)SQE_CTL_RMA_ADDR_OFFSET) & + (uint32_t)SQE_CTL_RMA_ADDR_BIT; + + if (sge->length <= UDMA_ATOMIC_LEN_8) + memcpy((void *)sqe_ctl + SQE_ATOMIC_DATA_FIELD, &wr->faa.operand, + sge->length); + else + memcpy((void *)sqe_ctl + SQE_ATOMIC_DATA_FIELD, + (void *)(uintptr_t)wr->faa.operand_addr, sge->length); + + return 0; +} + +static int udma_fill_normal_sge(struct udma_dev *dev, struct udma_sqe_ctl *sqe_ctl, + uint32_t max_inline_size, struct ubcore_jfs_wr *wr, + struct ubcore_tjetty *tjetty) +{ + switch (sqe_ctl->opcode) { + case UDMA_OPC_SEND: + case UDMA_OPC_SEND_WITH_IMM: + case UDMA_OPC_SEND_WITH_INVALID: + return udma_k_fill_send_sqe(dev, sqe_ctl, wr, tjetty, + max_inline_size); + case UDMA_OPC_WRITE: + return udma_k_fill_write_sqe(dev, sqe_ctl, wr, tjetty, max_inline_size); + case UDMA_OPC_WRITE_WITH_IMM: + return udma_k_fill_write_sqe(dev, sqe_ctl, wr, tjetty, + max_inline_size > SQE_WRITE_IMM_INLINE_SIZE ? + SQE_WRITE_IMM_INLINE_SIZE : max_inline_size); + case UDMA_OPC_READ: + return udma_k_fill_read_sqe(sqe_ctl, wr); + case UDMA_OPC_CAS: + return udma_k_fill_cas_sqe(dev, sqe_ctl, wr); + case UDMA_OPC_FAA: + return udma_k_fill_faa_sqe(dev, sqe_ctl, wr); + default: + return -EINVAL; + } +} + +static int udma_k_set_sqe(struct udma_sqe_ctl *sqe_ctl, struct ubcore_jfs_wr *wr, + struct udma_jetty_queue *sq, uint8_t opcode, + struct udma_dev *dev) +{ + struct udma_target_jetty *udma_tjetty; + struct ubcore_tjetty *tjetty; + int ret = 0; + + sqe_ctl->cqe = wr->flag.bs.complete_enable; + sqe_ctl->owner = (sq->pi & sq->buf.entry_cnt) == 0 ? 1 : 0; + sqe_ctl->opcode = opcode; + sqe_ctl->place_odr = wr->flag.bs.place_order; + + if (opcode == UDMA_OPC_NOP) + return 0; + + if (sq->trans_mode == UBCORE_TP_RC) + tjetty = sq->rc_tjetty; + else + tjetty = wr->tjetty; + + udma_tjetty = to_udma_tjetty(tjetty); + + sqe_ctl->tpn = tjetty->vtpn->vtpn; + sqe_ctl->fence = wr->flag.bs.fence; + sqe_ctl->comp_order = wr->flag.bs.comp_order; + sqe_ctl->se = wr->flag.bs.solicited_enable; + sqe_ctl->inline_en = wr->flag.bs.inline_flag; + sqe_ctl->rmt_jetty_type = tjetty->cfg.type; + memcpy(sqe_ctl->rmt_eid, &udma_tjetty->le_eid.raw, sizeof(uint8_t) * + UDMA_SQE_RMT_EID_SIZE); + + ret = udma_fill_normal_sge(dev, sqe_ctl, sq->max_inline_size, wr, tjetty); + if (ret) + dev_err(dev->dev, "Failed to fill normal sge, opcode :%u in wr.\n", + (uint8_t)wr->opcode); + + return ret; +} + +static bool udma_k_check_sge_num(uint8_t opcode, struct udma_jetty_queue *sq, + struct ubcore_jfs_wr *wr) +{ + switch (opcode) { + case UDMA_OPC_CAS: + case UDMA_OPC_FAA: + return sq->max_sge_num == 0; + case UDMA_OPC_READ: + return wr->rw.dst.num_sge > UDMA_JFS_MAX_SGE_READ || + wr->rw.dst.num_sge > sq->max_sge_num; + case UDMA_OPC_WRITE_WITH_IMM: + return wr->rw.src.num_sge > UDMA_JFS_MAX_SGE_WRITE_IMM || + wr->rw.src.num_sge > sq->max_sge_num; + case UDMA_OPC_SEND: + case UDMA_OPC_SEND_WITH_IMM: + case UDMA_OPC_SEND_WITH_INVALID: + return wr->send.src.num_sge > sq->max_sge_num; + default: + return wr->rw.src.num_sge > sq->max_sge_num; + } +} + +static void udma_copy_to_sq(struct udma_jetty_queue *sq, uint32_t wqebb_cnt, + struct udma_jfs_wqebb *tmp_sq) +{ + uint32_t remain = sq->buf.entry_cnt - (sq->pi & (sq->buf.entry_cnt - 1)); + uint32_t field_h; + uint32_t field_l; + + field_h = remain > wqebb_cnt ? wqebb_cnt : remain; + field_l = wqebb_cnt > field_h ? wqebb_cnt - field_h : 0; + + memcpy(sq->kva_curr, tmp_sq, field_h * sizeof(*tmp_sq)); + + if (field_l) + memcpy(sq->buf.kva, tmp_sq + field_h, field_l * sizeof(*tmp_sq)); +} + +static void *udma_k_inc_ptr_wrap(uint32_t sq_buf_size, uint32_t wqebb_size, + uint8_t *sq_base_addr, uint8_t *sq_buf_curr) +{ + uint8_t *sq_buf_end; + + sq_buf_end = (uint8_t *)(sq_buf_size + sq_base_addr); + + sq_buf_curr = ((sq_buf_curr + wqebb_size) < sq_buf_end) ? + (sq_buf_curr + wqebb_size) : sq_base_addr + (sq_buf_curr + + wqebb_size - sq_buf_end); + + return sq_buf_curr; +} + +static int udma_post_one_wr(struct udma_jetty_queue *sq, struct ubcore_jfs_wr *wr, + struct udma_dev *udma_dev, struct udma_sqe_ctl **wqe_addr, + bool *dwqe_enable) +{ + struct udma_jfs_wqebb tmp_sq[MAX_WQEBB_NUM] = {}; + uint32_t wqebb_cnt; + uint8_t opcode; + uint32_t i; + int ret; + + opcode = udma_get_jfs_opcode(wr->opcode); + if (unlikely(opcode == UDMA_OPC_INVALID)) { + dev_err(udma_dev->dev, "Invalid opcode :%u.\n", wr->opcode); + return -EINVAL; + } + + if (unlikely(udma_k_check_sge_num(opcode, sq, wr))) { + dev_err(udma_dev->dev, "WR sge num invalid.\n"); + return -EINVAL; + } + + ret = udma_k_set_sqe((struct udma_sqe_ctl *)(void *)tmp_sq, wr, sq, + opcode, udma_dev); + if (ret) + return ret; + + wqebb_cnt = get_wqebb_num((struct udma_sqe_ctl *)(void *)tmp_sq); + if (wqebb_cnt == 1 && !!(udma_dev->caps.feature & UDMA_CAP_FEATURE_DIRECT_WQE)) + *dwqe_enable = true; + + if (to_check_sq_overflow(sq, wqebb_cnt)) { + dev_err(udma_dev->dev, "JFS overflow, wqebb_cnt:%u.\n", wqebb_cnt); + return -ENOMEM; + } + + udma_copy_to_sq(sq, wqebb_cnt, tmp_sq); + + *wqe_addr = (struct udma_sqe_ctl *)sq->kva_curr; + + sq->kva_curr = udma_k_inc_ptr_wrap(sq->buf.entry_cnt * sq->buf.entry_size, + wqebb_cnt * sq->buf.entry_size, + (uint8_t *)sq->buf.kva, + (uint8_t *)sq->kva_curr); + + for (i = 0; i < wqebb_cnt; i++) + sq->wrid[(sq->pi + i) & (sq->buf.entry_cnt - 1)] = wr->user_ctx; + + sq->pi += wqebb_cnt; + + return 0; +} + +static inline void udma_k_update_sq_db(struct udma_jetty_queue *sq) +{ + uint32_t *db_addr = sq->db_addr; + *db_addr = sq->pi; +} + +#ifdef ST64B +static void st64b(uint64_t *src, uint64_t *dst) +{ + asm volatile ( + "mov x9, %0\n" + "mov x10, %1\n" + "ldr x0, [x9]\n" + "ldr x1, [x9, #8]\n" + "ldr x2, [x9, #16]\n" + "ldr x3, [x9, #24]\n" + "ldr x4, [x9, #32]\n" + "ldr x5, [x9, #40]\n" + "ldr x6, [x9, #48]\n" + "ldr x7, [x9, #56]\n" + ".inst 0xf83f9140\n" + ::"r" (src), "r"(dst):"cc", "memory" + ); +} +#endif + +static void udma_write_dsqe(struct udma_jetty_queue *sq, + struct udma_sqe_ctl *ctrl) +{ +#define DWQE_SIZE 8 + int i; + + ctrl->sqe_bb_idx = sq->pi; + +#ifdef ST64B + st64b(((uint64_t *)ctrl), (uint64_t *)sq->dwqe_addr); +#else + for (i = 0; i < DWQE_SIZE; i++) + writeq_relaxed(*((uint64_t *)ctrl + i), + (uint64_t *)sq->dwqe_addr + i); +#endif +} + +/* thanks to drivers/infiniband/hw/bnxt_re/ib_verbs.c */ +int udma_post_sq_wr(struct udma_dev *udma_dev, struct udma_jetty_queue *sq, + struct ubcore_jfs_wr *wr, struct ubcore_jfs_wr **bad_wr) +{ + struct udma_sqe_ctl *wqe_addr; + bool dwqe_enable = false; + struct ubcore_jfs_wr *it; + unsigned long flags; + int wr_cnt = 0; + int ret = 0; + + if (!sq->lock_free) + spin_lock_irqsave(&sq->lock, flags); + + for (it = wr; it != NULL; it = (struct ubcore_jfs_wr *)(void *)it->next) { + ret = udma_post_one_wr(sq, it, udma_dev, &wqe_addr, &dwqe_enable); + if (ret) { + *bad_wr = it; + goto err_post_wr; + } + wr_cnt++; + } + +err_post_wr: + if (likely(wr_cnt && udma_dev->status != UDMA_SUSPEND)) { + wmb(); /* set sqe before doorbell */ + if (wr_cnt == 1 && dwqe_enable && (sq->pi - sq->ci == 1)) + udma_write_dsqe(sq, wqe_addr); + else + udma_k_update_sq_db(sq); + } + + if (!sq->lock_free) + spin_unlock_irqrestore(&sq->lock, flags); + + return ret; +} + +int udma_post_jfs_wr(struct ubcore_jfs *jfs, struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr) +{ + struct udma_dev *udma_dev = to_udma_dev(jfs->ub_dev); + struct udma_jfs *udma_jfs = to_udma_jfs(jfs); + int ret; + + ret = udma_post_sq_wr(udma_dev, &udma_jfs->sq, wr, bad_wr); + if (ret) + dev_err(udma_dev->dev, "Failed to post jfs wr, sq_id = %u.\n", + udma_jfs->sq.id); + + return ret; +} diff --git a/drivers/ub/urma/hw/udma/udma_jfs.h b/drivers/ub/urma/hw/udma/udma_jfs.h new file mode 100644 index 0000000000000000000000000000000000000000..d3a29f2a68a04cfb5cf7c0b9bf272e79345f4847 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_jfs.h @@ -0,0 +1,160 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_JFS_H__ +#define __UDMA_JFS_H__ + +#include "udma_common.h" + +#define MAX_WQEBB_NUM 4 +#define UDMA_SQE_RMT_EID_SIZE 16 +#define SQE_WRITE_IMM_CTL_LEN 64 +#define SQE_NORMAL_CTL_LEN 48 +#define ATOMIC_WQEBB_CNT 2 +#define NOP_WQEBB_CNT 1 +#define UDMA_JFS_WQEBB_SIZE 64 +#define UDMA_JFS_SGE_SIZE 16 +#define UDMA_JFS_MAX_SGE_READ 6 +#define UDMA_JFS_MAX_SGE_WRITE_IMM 12 +#define UDMA_ATOMIC_SGE_NUM 1 +#define UDMA_ATOMIC_LEN_4 4 +#define UDMA_ATOMIC_LEN_8 8 +#define UDMA_ATOMIC_LEN_16 16 +#define SQE_CTL_RMA_ADDR_OFFSET 32 +#define SQE_CTL_RMA_ADDR_BIT GENMASK(31, 0) +#define SQE_ATOMIC_DATA_FIELD 64 +#define SQE_SEND_IMM_FIELD 40 +#define WRITE_IMM_TOKEN_FIELD 56 +#define SQE_WRITE_IMM_FIELD 48 + +#define SQE_WRITE_NOTIFY_CTL_LEN 80 +#define SQE_WRITE_IMM_INLINE_SIZE 192 + +#define UINT8_MAX 0xff + +enum udma_jfs_type { + UDMA_NORMAL_JFS_TYPE, + UDMA_KERNEL_STARS_JFS_TYPE, +}; + +struct udma_jfs { + struct ubcore_jfs ubcore_jfs; + struct udma_jetty_queue sq; + uint64_t jfs_addr; + refcount_t ae_refcount; + struct completion ae_comp; + uint32_t mode; + bool pi_type; + bool ue_rx_closed; +}; + +/* thanks to include/rdma/ib_verbs.h */ +enum udma_sq_opcode { + UDMA_OPC_SEND, + UDMA_OPC_SEND_WITH_IMM, + UDMA_OPC_SEND_WITH_INVALID, + UDMA_OPC_WRITE, + UDMA_OPC_WRITE_WITH_IMM, + UDMA_OPC_READ = 0x6, + UDMA_OPC_CAS, + UDMA_OPC_FAA = 0xb, + UDMA_OPC_NOP = 0x11, + UDMA_OPC_INVALID = 0x12, +}; + +struct udma_jfs_wqebb { + uint32_t value[16]; +}; + +struct udma_sqe_ctl { + uint32_t sqe_bb_idx : 16; + uint32_t place_odr : 2; + uint32_t comp_order : 1; + uint32_t fence : 1; + uint32_t se : 1; + uint32_t cqe : 1; + uint32_t inline_en : 1; + uint32_t rsv : 5; + uint32_t token_en : 1; + uint32_t rmt_jetty_type : 2; + uint32_t owner : 1; + uint32_t target_hint : 8; + uint32_t opcode : 8; + uint32_t rsv1 : 6; + uint32_t inline_msg_len : 10; + uint32_t tpn : 24; + uint32_t sge_num : 8; + uint32_t rmt_obj_id : 20; + uint32_t rsv2 : 12; + uint8_t rmt_eid[UDMA_SQE_RMT_EID_SIZE]; + uint32_t rmt_token_value; + uint32_t rsv3; + uint32_t rmt_addr_l_or_token_id; + uint32_t rmt_addr_h_or_token_value; +}; + +struct udma_normal_sge { + uint32_t length; + uint32_t token_id; + uint64_t va; +}; + +struct udma_token_info { + uint32_t token_id : 20; + uint32_t rsv : 12; + uint32_t token_value; +}; + +static inline struct udma_jfs *to_udma_jfs(struct ubcore_jfs *jfs) +{ + return container_of(jfs, struct udma_jfs, ubcore_jfs); +} + +static inline struct udma_jfs *to_udma_jfs_from_queue(struct udma_jetty_queue *queue) +{ + return container_of(queue, struct udma_jfs, sq); +} + +static inline bool to_check_sq_overflow(struct udma_jetty_queue *sq, + uint32_t wqebb_cnt) +{ + return sq->pi - sq->ci + wqebb_cnt > sq->buf.entry_cnt; +} + +static inline uint32_t sq_cal_wqebb_num(uint32_t sqe_ctl_len, uint32_t sge_num) +{ + return (sqe_ctl_len + (sge_num - 1) * UDMA_JFS_SGE_SIZE) / + UDMA_JFS_WQEBB_SIZE + 1; +} + +static inline uint32_t get_ctl_len(uint8_t opcode) +{ + return opcode == UDMA_OPC_WRITE_WITH_IMM ? SQE_WRITE_IMM_CTL_LEN : SQE_NORMAL_CTL_LEN; +} + +struct ubcore_jfs *udma_create_jfs(struct ubcore_device *ub_dev, + struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jfs(struct ubcore_jfs *jfs); +int udma_destroy_jfs_batch(struct ubcore_jfs **jfs_arr, int jfs_num, int *bad_jfs_index); +int udma_alloc_u_sq_buf(struct udma_dev *dev, struct udma_jetty_queue *sq, + struct udma_create_jetty_ucmd *ucmd); +int udma_alloc_k_sq_buf(struct udma_dev *dev, struct udma_jetty_queue *sq, + struct ubcore_jfs_cfg *jfs_cfg); +void udma_free_sq_buf(struct udma_dev *dev, struct udma_jetty_queue *sq); +int udma_modify_jfs(struct ubcore_jfs *jfs, struct ubcore_jfs_attr *attr, + struct ubcore_udata *udata); +int udma_flush_jfs(struct ubcore_jfs *jfs, int cr_cnt, struct ubcore_cr *cr); +int udma_post_sq_wr(struct udma_dev *udma_dev, struct udma_jetty_queue *sq, + struct ubcore_jfs_wr *wr, struct ubcore_jfs_wr **bad_wr); +int udma_post_jfs_wr(struct ubcore_jfs *jfs, struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr); +void udma_flush_sq(struct udma_dev *udma_dev, + struct udma_jetty_queue *sq, struct ubcore_cr *cr); +void udma_dfx_store_jfs_id(struct udma_dev *udma_dev, struct udma_jfs *udma_jfs); +void udma_init_jfsc(struct udma_dev *dev, struct ubcore_jfs_cfg *cfg, + struct udma_jfs *jfs, void *mb_buf); +int udma_verify_jfs_param(struct udma_dev *dev, struct ubcore_jfs_cfg *cfg, + bool enable_stars); + +#endif /* __UDMA_JFS_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_main.c b/drivers/ub/urma/hw/udma/udma_main.c new file mode 100644 index 0000000000000000000000000000000000000000..44a93fd000b049bf4579130c265664d97a77b23e --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_main.c @@ -0,0 +1,1203 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt +#define pr_fmt(fmt) "UDMA: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "udma_dev.h" +#include "udma_eq.h" +#include "udma_segment.h" +#include "udma_jfs.h" +#include "udma_jfc.h" +#include "udma_jfr.h" +#include "udma_cmd.h" +#include "udma_ctx.h" +#include "udma_rct.h" +#include "udma_tid.h" +#include "udma_dfx.h" +#include "udma_eid.h" +#include "udma_debugfs.h" +#include "udma_common.h" +#include "udma_ctrlq_tp.h" + +bool cqe_mode = true; +bool is_rmmod; +static DEFINE_MUTEX(udma_reset_mutex); +uint32_t jfr_sleep_time = 1000; +uint32_t jfc_arm_mode; +bool dump_aux_info; + +static const struct auxiliary_device_id udma_id_table[] = { + { + .name = UBASE_ADEV_NAME ".udma", + }, + {}, +}; +MODULE_DEVICE_TABLE(auxiliary, udma_id_table); + +static int udma_set_eth_device_speed(struct ubcore_device *dev, + struct ubcore_device_status *dev_status, uint32_t speed) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + + switch (speed) { + case SPEED_400G: + dev_status->port_status[0].active_speed = UBCORE_SP_400G; + break; + case SPEED_200G: + dev_status->port_status[0].active_speed = UBCORE_SP_200G; + break; + case SPEED_100G: + dev_status->port_status[0].active_speed = UBCORE_SP_100G; + break; + case SPEED_50G: + dev_status->port_status[0].active_speed = UBCORE_SP_50G; + break; + case SPEED_25G: + dev_status->port_status[0].active_speed = UBCORE_SP_25G; + break; + default: + dev_err(udma_dev->dev, "invalid port speed(%u) in UBOE mode.\n", speed); + return -EINVAL; + } + + return 0; +} + +static int udma_query_device_status(struct ubcore_device *dev, + struct ubcore_device_status *dev_status) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_cmd_port_info port_info; + struct ubase_cmd_buf in, out; + int ret; + + dev_status->port_status[0].state = UBCORE_PORT_ACTIVE; + dev_status->port_status[0].active_mtu = UBCORE_MTU_4096; + + udma_fill_buf(&in, UDMA_CMD_QUERY_PORT_INFO, true, 0, NULL); + udma_fill_buf(&out, UDMA_CMD_QUERY_PORT_INFO, true, + sizeof(port_info), (void *)&port_info); + ret = ubase_cmd_send_inout(udma_dev->comdev.adev, &in, &out); + if (ret) { + dev_err(udma_dev->dev, "failed to query speed, ret = %d.\n", ret); + return -EINVAL; + } + + dev_status->port_status[0].active_width = (enum ubcore_link_width)port_info.lanes; + + if (!ubase_adev_ubl_supported(udma_dev->comdev.adev)) + return udma_set_eth_device_speed(dev, dev_status, port_info.speed); + + if (port_info.speed == SPEED_200G) { + dev_status->port_status[0].active_speed = UBCORE_SP_200G; + } else if (port_info.speed == SPEED_400G) { + dev_status->port_status[0].active_speed = UBCORE_SP_400G; + } else { + dev_err(udma_dev->dev, "invalid port speed = %u.\n", port_info.speed); + ret = -EINVAL; + } + + return ret; +} + +static void udma_set_dev_caps(struct ubcore_device_attr *attr, struct udma_dev *udma_dev) +{ + attr->dev_cap.max_jfs_depth = udma_dev->caps.jfs.depth; + attr->dev_cap.max_jfr_depth = udma_dev->caps.jfr.depth; + attr->dev_cap.max_jfc_depth = udma_dev->caps.jfc.depth; + attr->dev_cap.max_jfs = udma_dev->caps.jfs.max_cnt + + udma_dev->caps.public_jetty.max_cnt + + udma_dev->caps.user_ctrl_normal_jetty.max_cnt; + attr->dev_cap.max_jfr = udma_dev->caps.jfr.max_cnt; + attr->dev_cap.max_jfc = udma_dev->caps.jfc.max_cnt; + attr->dev_cap.max_jetty = udma_dev->caps.jetty.max_cnt + + udma_dev->caps.public_jetty.max_cnt + + udma_dev->caps.user_ctrl_normal_jetty.max_cnt; + attr->dev_cap.max_jetty_grp = udma_dev->caps.jetty_grp.max_cnt; + attr->dev_cap.max_jetty_in_jetty_grp = udma_dev->caps.jetty_in_grp; + attr->dev_cap.max_jfs_rsge = udma_dev->caps.jfs_rsge; + attr->dev_cap.max_jfs_sge = udma_dev->caps.jfs_sge; + attr->dev_cap.max_jfs_inline_size = udma_dev->caps.jfs_inline_sz; + attr->dev_cap.max_jfr_sge = udma_dev->caps.jfr_sge; + attr->dev_cap.max_msg_size = udma_dev->caps.max_msg_len; + attr->dev_cap.trans_mode = udma_dev->caps.trans_mode; + attr->port_cnt = udma_dev->caps.port_num; + attr->dev_cap.ceq_cnt = udma_dev->caps.comp_vector_cnt; + attr->dev_cap.max_ue_cnt = udma_dev->caps.ue_cnt; + attr->dev_cap.max_rc = udma_dev->caps.rc_queue_num; + attr->dev_cap.max_rc_depth = udma_dev->caps.rc_queue_depth; + attr->dev_cap.max_eid_cnt = udma_dev->caps.seid.max_cnt; + attr->dev_cap.feature.bs.jfc_inline = (udma_dev->caps.feature & + UDMA_CAP_FEATURE_JFC_INLINE) ? 1 : 0; + attr->dev_cap.max_read_size = udma_dev->caps.max_read_size; + attr->dev_cap.max_write_size = udma_dev->caps.max_write_size; + attr->dev_cap.max_cas_size = udma_dev->caps.max_cas_size; + attr->dev_cap.max_fetch_and_add_size = udma_dev->caps.max_fetch_and_add_size; + attr->dev_cap.atomic_feat.value = udma_dev->caps.atomic_feat; +} + +static int udma_query_device_attr(struct ubcore_device *dev, + struct ubcore_device_attr *attr) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + + udma_set_dev_caps(attr, udma_dev); + attr->ue_idx = udma_dev->caps.ue_id; + attr->port_attr[0].max_mtu = UBCORE_MTU_4096; + attr->reserved_jetty_id_max = udma_dev->caps.public_jetty.max_cnt - 1; + + return 0; +} + +static int udma_query_stats(struct ubcore_device *dev, struct ubcore_stats_key *key, + struct ubcore_stats_val *val) +{ + struct ubcore_stats_com_val *com_val = (struct ubcore_stats_com_val *)val->addr; + struct udma_dev *udma_dev = to_udma_dev(dev); + struct ubase_ub_dl_stats dl_stats = {}; + int ret; + + ret = ubase_get_ub_port_stats(udma_dev->comdev.adev, + udma_dev->port_logic_id, &dl_stats); + if (ret) { + dev_err(udma_dev->dev, "failed to query port stats, ret = %d.\n", ret); + return ret; + } + + com_val->tx_pkt = dl_stats.dl_tx_busi_pkt_num; + com_val->rx_pkt = dl_stats.dl_rx_busi_pkt_num; + com_val->rx_pkt_err = 0; + com_val->tx_pkt_err = 0; + com_val->tx_bytes = 0; + com_val->rx_bytes = 0; + + return ret; +} + +static void udma_disassociate_ucontext(struct ubcore_ucontext *uctx) +{ +} + +static struct ubcore_ops g_dev_ops = { + .owner = THIS_MODULE, + .abi_version = 0, + .query_device_attr = udma_query_device_attr, + .query_device_status = udma_query_device_status, + .query_res = udma_query_res, + .config_device = udma_config_device, + .alloc_ucontext = udma_alloc_ucontext, + .free_ucontext = udma_free_ucontext, + .mmap = udma_mmap, + .alloc_token_id = udma_alloc_tid, + .free_token_id = udma_free_tid, + .register_seg = udma_register_seg, + .unregister_seg = udma_unregister_seg, + .import_seg = udma_import_seg, + .unimport_seg = udma_unimport_seg, + .create_jfc = udma_create_jfc, + .modify_jfc = udma_modify_jfc, + .destroy_jfc = udma_destroy_jfc, + .rearm_jfc = udma_rearm_jfc, + .create_jfs = udma_create_jfs, + .modify_jfs = udma_modify_jfs, + .query_jfs = udma_query_jfs, + .flush_jfs = udma_flush_jfs, + .destroy_jfs = udma_destroy_jfs, + .destroy_jfs_batch = udma_destroy_jfs_batch, + .create_jfr = udma_create_jfr, + .modify_jfr = udma_modify_jfr, + .query_jfr = udma_query_jfr, + .destroy_jfr = udma_destroy_jfr, + .destroy_jfr_batch = udma_destroy_jfr_batch, + .import_jfr_ex = udma_import_jfr_ex, + .unimport_jfr = udma_unimport_jfr, + .create_jetty = udma_create_jetty, + .modify_jetty = udma_modify_jetty, + .query_jetty = udma_query_jetty, + .flush_jetty = udma_flush_jetty, + .destroy_jetty = udma_destroy_jetty, + .destroy_jetty_batch = udma_destroy_jetty_batch, + .import_jetty_ex = udma_import_jetty_ex, + .unimport_jetty = udma_unimport_jetty, + .bind_jetty_ex = udma_bind_jetty_ex, + .unbind_jetty = udma_unbind_jetty, + .create_jetty_grp = udma_create_jetty_grp, + .delete_jetty_grp = udma_delete_jetty_grp, + .get_tp_list = udma_get_tp_list, + .set_tp_attr = udma_set_tp_attr, + .get_tp_attr = udma_get_tp_attr, + .active_tp = udma_active_tp, + .deactive_tp = udma_deactive_tp, + .user_ctl = udma_user_ctl, + .post_jfs_wr = udma_post_jfs_wr, + .post_jfr_wr = udma_post_jfr_wr, + .post_jetty_send_wr = udma_post_jetty_send_wr, + .post_jetty_recv_wr = udma_post_jetty_recv_wr, + .poll_jfc = udma_poll_jfc, + .query_stats = udma_query_stats, + .query_ue_idx = udma_query_ue_idx, + .disassociate_ucontext = udma_disassociate_ucontext, +}; + +static void udma_uninit_group_table(struct udma_dev *dev, struct udma_group_table *table) +{ + if (!xa_empty(&table->xa)) + dev_err(dev->dev, "table is not empty.\n"); + xa_destroy(&table->xa); + + vfree(table->bitmap_table.bit); + table->bitmap_table.bit = NULL; +} + +static void udma_destroy_tp_ue_idx_table(struct udma_dev *udma_dev) +{ + struct udma_ue_idx_table *tp_ue_idx_info; + unsigned long index = 0; + + xa_lock(&udma_dev->tpn_ue_idx_table); + if (!xa_empty(&udma_dev->tpn_ue_idx_table)) { + xa_for_each(&udma_dev->tpn_ue_idx_table, index, tp_ue_idx_info) { + __xa_erase(&udma_dev->tpn_ue_idx_table, index); + kfree(tp_ue_idx_info); + tp_ue_idx_info = NULL; + } + } + + xa_unlock(&udma_dev->tpn_ue_idx_table); + xa_destroy(&udma_dev->tpn_ue_idx_table); +} + +void udma_destroy_tables(struct udma_dev *udma_dev) +{ + udma_ctrlq_destroy_tpid_list(udma_dev, &udma_dev->ctrlq_tpid_table, false); + udma_destroy_eid_table(udma_dev); + mutex_destroy(&udma_dev->disable_ue_rx_mutex); + if (!ida_is_empty(&udma_dev->rsvd_jetty_ida_table.ida)) + dev_err(udma_dev->dev, + "IDA not empty in clean up rsvd jetty id table.\n"); + ida_destroy(&udma_dev->rsvd_jetty_ida_table.ida); + + if (!xa_empty(&udma_dev->crq_nb_table)) + dev_err(udma_dev->dev, "crq nb table is not empty.\n"); + xa_destroy(&udma_dev->crq_nb_table); + + udma_destroy_tp_ue_idx_table(udma_dev); + udma_destroy_npu_cb_table(udma_dev); + + if (!xa_empty(&udma_dev->ksva_table)) + dev_err(udma_dev->dev, "ksva table is not empty.\n"); + xa_destroy(&udma_dev->ksva_table); + mutex_destroy(&udma_dev->ksva_mutex); + udma_destroy_udma_table(udma_dev, &udma_dev->jetty_grp_table, "JettyGroup"); + udma_destroy_udma_table(udma_dev, &udma_dev->jfc_table, "JFC"); + udma_destroy_udma_table(udma_dev, &udma_dev->jfr_table, "JFR"); + udma_uninit_group_table(udma_dev, &udma_dev->jetty_table); +} + +static int udma_init_group_table(struct udma_dev *udma_dev, struct udma_group_table *table, + uint32_t max, uint32_t min, uint32_t num_per_group) +{ + struct udma_group_bitmap *bitmap_table; + int i; + + bitmap_table = &table->bitmap_table; + if (max < min) { + dev_err(udma_dev->dev, + "max value is less than min value when init group bitmap.\n"); + return -EINVAL; + } + + bitmap_table->max = max; + bitmap_table->min = min; + bitmap_table->grp_next = min; + bitmap_table->n_bits = max - min + 1; + bitmap_table->bitmap_cnt = ALIGN(bitmap_table->n_bits, num_per_group) / + num_per_group; + bitmap_table->bit = vmalloc(bitmap_table->bitmap_cnt * sizeof(uint32_t)); + if (!bitmap_table->bit) { + dev_err(udma_dev->dev, "failed to alloc jetty bitmap.\n"); + return -ENOMEM; + } + + for (i = 0; i < bitmap_table->bitmap_cnt; ++i) + bitmap_table->bit[i] = ~(0U); + + spin_lock_init(&bitmap_table->lock); + xa_init(&table->xa); + + return 0; +} + +static void udma_init_managed_by_ctrl_cpu_table(struct udma_dev *udma_dev) +{ + mutex_init(&udma_dev->eid_mutex); + xa_init(&udma_dev->eid_table); + xa_init(&udma_dev->ctrlq_tpid_table); +} + +int udma_init_tables(struct udma_dev *udma_dev) +{ + int ret; + + ret = udma_init_group_table(udma_dev, &udma_dev->jetty_table, + udma_dev->caps.jetty.max_cnt + + udma_dev->caps.jetty.start_idx - 1, + udma_dev->caps.jetty.start_idx, + NUM_JETTY_PER_GROUP); + if (ret) { + dev_err(udma_dev->dev, + "failed to init jetty table when start_idx = %u, and max_cnt = %u.\n", + udma_dev->caps.jetty.start_idx, udma_dev->caps.jetty.max_cnt); + return ret; + } + + udma_init_udma_table(&udma_dev->jfr_table, udma_dev->caps.jfr.max_cnt + + udma_dev->caps.jfr.start_idx - 1, udma_dev->caps.jfr.start_idx); + udma_init_udma_table(&udma_dev->jfc_table, udma_dev->caps.jfc.max_cnt + + udma_dev->caps.jfc.start_idx - 1, udma_dev->caps.jfc.start_idx); + udma_init_udma_table(&udma_dev->jetty_grp_table, udma_dev->caps.jetty_grp.max_cnt + + udma_dev->caps.jetty_grp.start_idx - 1, + udma_dev->caps.jetty_grp.start_idx); + udma_init_udma_table_mutex(&udma_dev->ksva_table, &udma_dev->ksva_mutex); + udma_init_udma_table_mutex(&udma_dev->npu_nb_table, &udma_dev->npu_nb_mutex); + xa_init(&udma_dev->tpn_ue_idx_table); + xa_init(&udma_dev->crq_nb_table); + ida_init(&udma_dev->rsvd_jetty_ida_table.ida); + mutex_init(&udma_dev->disable_ue_rx_mutex); + udma_init_managed_by_ctrl_cpu_table(udma_dev); + + return 0; +} + +static void udma_free_rct(struct udma_dev *udev) +{ + uint32_t min = udev->rc_table.ida_table.min; + uint32_t max = udev->rc_table.ida_table.max; + uint32_t i; + + if (test_and_clear_bit(RCT_INIT_FLAG, &udev->caps.init_flag)) + for (i = min; i < max; i++) + udma_free_rc_queue(udev, i); +} + +static void udma_unset_ubcore_dev(struct udma_dev *udma_dev) +{ + struct ubcore_device *ub_dev = &udma_dev->ub_dev; + + ubcore_unregister_device(ub_dev); + udma_free_rct(udma_dev); +} + +static int udma_set_ubcore_dev(struct udma_dev *udma_dev) +{ + struct ubcore_device *ub_dev = &udma_dev->ub_dev; + int ret; + + ub_dev->transport_type = UBCORE_TRANSPORT_UB; + ub_dev->ops = &g_dev_ops; + ub_dev->dev.parent = udma_dev->dev; + ub_dev->dma_dev = ub_dev->dev.parent; + ub_dev->attr.dev_cap.feature.value = udma_dev->caps.feature; + + scnprintf(udma_dev->dev_name, UBCORE_MAX_DEV_NAME, "udma%hu", udma_dev->adev_id); + strscpy(ub_dev->dev_name, udma_dev->dev_name, UBCORE_MAX_DEV_NAME); + scnprintf(ub_dev->ops->driver_name, UBCORE_MAX_DRIVER_NAME, "udma"); + + ret = ubcore_register_device(ub_dev); + if (ret) + dev_err(udma_dev->dev, "failed to register udma_dev to ubcore, ret is %d.\n", ret); + + return ret; +} + +static void udma_dump_jetty_id_range(struct udma_dev *udma_dev) +{ +#define UDMA_JETTY_CNT 6 + const char *jetty_name[UDMA_JETTY_CNT] = { + "public", + "ccu", + "hdc", + "cache_lock", + "user_ctrl_normal", + "urma_normal", + }; + struct udma_res *jetty_res_list[UDMA_JETTY_CNT] = { + &udma_dev->caps.public_jetty, + &udma_dev->caps.ccu_jetty, + &udma_dev->caps.hdc_jetty, + &udma_dev->caps.stars_jetty, + &udma_dev->caps.user_ctrl_normal_jetty, + &udma_dev->caps.jetty, + }; + uint32_t i; + + for (i = 0; i < UDMA_JETTY_CNT; i++) + dev_info(udma_dev->dev, "%s jetty start_idx=%u, max_cnt=%u\n", + jetty_name[i], jetty_res_list[i]->start_idx, + jetty_res_list[i]->max_cnt); +} + +static void udma_get_jetty_id_range(struct udma_dev *udma_dev, + struct udma_cmd_ue_resource *cmd) +{ + udma_dev->caps.public_jetty.start_idx = cmd->well_known_jetty_start; + udma_dev->caps.public_jetty.max_cnt = cmd->well_known_jetty_num; + + udma_dev->caps.ccu_jetty.start_idx = cmd->ccu_jetty_start; + udma_dev->caps.ccu_jetty.max_cnt = cmd->ccu_jetty_num; + udma_dev->caps.ccu_jetty.next_idx = udma_dev->caps.ccu_jetty.start_idx; + + udma_dev->caps.hdc_jetty.start_idx = cmd->drv_jetty_start; + udma_dev->caps.hdc_jetty.max_cnt = cmd->drv_jetty_num; + + udma_dev->caps.stars_jetty.start_idx = cmd->cache_lock_jetty_start; + udma_dev->caps.stars_jetty.max_cnt = cmd->cache_lock_jetty_num; + udma_dev->caps.stars_jetty.next_idx = udma_dev->caps.stars_jetty.start_idx; + + udma_dev->caps.user_ctrl_normal_jetty.start_idx = cmd->normal_jetty_start; + udma_dev->caps.user_ctrl_normal_jetty.max_cnt = cmd->normal_jetty_num; + udma_dev->caps.user_ctrl_normal_jetty.next_idx = + udma_dev->caps.user_ctrl_normal_jetty.start_idx; + + udma_dev->caps.jetty.start_idx = cmd->standard_jetty_start; + udma_dev->caps.jetty.max_cnt = cmd->standard_jetty_num; + + udma_dev->caps.rsvd_jetty_cnt = udma_dev->caps.public_jetty.max_cnt + + udma_dev->caps.ccu_jetty.max_cnt + + udma_dev->caps.hdc_jetty.max_cnt + + udma_dev->caps.stars_jetty.max_cnt + + udma_dev->caps.user_ctrl_normal_jetty.max_cnt; + + if (debug_switch) + udma_dump_jetty_id_range(udma_dev); +} + +static int query_caps_from_firmware(struct udma_dev *udma_dev) +{ + struct udma_cmd_ue_resource cmd = {}; + int ret; + + ret = udma_cmd_query_hw_resource(udma_dev, (void *)&cmd); + if (ret) { + dev_err(udma_dev->dev, "fail to query hw resource from FW %d\n", ret); + return ret; + } + + udma_dev->caps.jfs_sge = cmd.jfs_sge; + udma_dev->caps.jfs_rsge = cmd.jfs_rsge; + udma_dev->caps.jfr_sge = cmd.jfr_sge; + udma_dev->caps.jfs_inline_sz = cmd.max_jfs_inline_sz; + udma_dev->caps.jetty_grp.max_cnt = cmd.jetty_grp_num; + udma_dev->caps.trans_mode = cmd.trans_mode; + udma_dev->caps.seid.size = cmd.seid_upi_tbl_sz; + udma_dev->caps.seid.max_cnt = cmd.seid_upi_tbl_num; + udma_dev->caps.port_num = cmd.port_num; + udma_dev->caps.max_read_size = cmd.max_read_size; + udma_dev->caps.max_write_size = cmd.max_write_size; + udma_dev->caps.max_cas_size = cmd.max_cas_size; + udma_dev->caps.max_fetch_and_add_size = cmd.max_fetch_and_add_size; + udma_dev->caps.atomic_feat = cmd.atomic_feat; + + udma_get_jetty_id_range(udma_dev, &cmd); + + udma_dev->caps.feature = cmd.cap_info; + udma_dev->caps.ue_cnt = cmd.ue_cnt >= UDMA_DEV_UE_NUM ? + UDMA_DEV_UE_NUM - 1 : cmd.ue_cnt; + udma_dev->caps.ue_id = cmd.ue_id; + udma_dev->is_ue = !!(cmd.ue_id); + + return 0; +} + +static void get_dev_caps_from_ubase(struct udma_dev *udma_dev) +{ + struct ubase_caps *ubase_caps; + + ubase_caps = ubase_get_dev_caps(udma_dev->comdev.adev); + + udma_dev->caps.comp_vector_cnt = ubase_caps->num_ceq_vectors; + udma_dev->caps.ack_queue_num = ubase_caps->ack_queue_num; + + udma_dev->chip_id = ubase_caps->chip_id; + udma_dev->die_id = ubase_caps->die_id; + udma_dev->port_id = ubase_caps->io_port_id; + udma_dev->port_logic_id = ubase_caps->io_port_logic_id; + udma_dev->ue_id = ubase_caps->ue_id; +} + +static int udma_construct_qos_param(struct udma_dev *dev) +{ + struct ubase_adev_qos *qos_info; + uint8_t i; + + qos_info = ubase_get_adev_qos(dev->comdev.adev); + if (!qos_info) { + dev_err(dev->dev, "cannot get qos information from ubase.\n"); + return -EINVAL; + } + + dev->udma_tp_sl_num = qos_info->tp_sl_num; + dev->udma_ctp_sl_num = qos_info->ctp_sl_num; + dev->unic_sl_num = qos_info->nic_sl_num; + dev->udma_tp_resp_vl_off = qos_info->tp_resp_vl_offset; + dev->udma_total_sl_num = dev->udma_tp_sl_num + dev->udma_ctp_sl_num; + if (dev->udma_total_sl_num > UDMA_MAX_SL_NUM) { + dev_err(dev->dev, + "total sl num is invalid, tp sl num is %u, ctp sl num is %u.\n", + dev->udma_tp_sl_num, dev->udma_ctp_sl_num); + return -EINVAL; + } + + (void)memcpy(dev->udma_tp_sl, + qos_info->tp_sl, sizeof(u8) * qos_info->tp_sl_num); + (void)memcpy(dev->udma_ctp_sl, + qos_info->ctp_sl, sizeof(u8) * qos_info->ctp_sl_num); + (void)memcpy(dev->unic_sl, + qos_info->nic_sl, sizeof(u8) * qos_info->nic_sl_num); + (void)memcpy(dev->udma_sl, + qos_info->tp_sl, sizeof(u8) * qos_info->tp_sl_num); + + for (i = 0; i < qos_info->ctp_sl_num; i++) + dev->udma_sl[qos_info->tp_sl_num + i] = qos_info->ctp_sl[i]; + + return 0; +} + +static void cal_max_2m_num(struct udma_dev *dev) +{ + uint32_t jfs_pg = ALIGN(dev->caps.jfs.depth * MAX_WQEBB_IN_SQE * + UDMA_JFS_WQEBB_SIZE, UDMA_HUGEPAGE_SIZE) >> UDMA_HUGEPAGE_SHIFT; + uint32_t jfr_pg = ALIGN(dev->caps.jfr.depth * dev->caps.jfr_sge * + UDMA_SGE_SIZE, UDMA_HUGEPAGE_SIZE) >> UDMA_HUGEPAGE_SHIFT; + uint32_t jfc_pg = ALIGN(dev->caps.jfc.depth * dev->caps.cqe_size, + UDMA_HUGEPAGE_SIZE) >> UDMA_HUGEPAGE_SHIFT; + + dev->total_hugepage_num = + (dev->caps.jetty.start_idx + dev->caps.jetty.max_cnt) * jfs_pg + + dev->caps.jfr.max_cnt * jfr_pg + dev->caps.jfc.max_cnt * jfc_pg; +} + +static int udma_set_hw_caps(struct udma_dev *udma_dev) +{ +#define MAX_MSG_LEN 0x10000 +#define RC_QUEUE_ENTRY_SIZE 64 + struct ubase_adev_caps *a_caps; + uint32_t jetty_grp_cnt; + int ret; + + get_dev_caps_from_ubase(udma_dev); + + ret = query_caps_from_firmware(udma_dev); + if (ret) + return ret; + + a_caps = ubase_get_udma_caps(udma_dev->comdev.adev); + udma_dev->caps.jfs.max_cnt = a_caps->jfs.max_cnt; + udma_dev->caps.jfs.depth = a_caps->jfs.depth / MAX_WQEBB_IN_SQE; + udma_dev->caps.jfs.start_idx = a_caps->jfs.start_idx; + udma_dev->caps.jfr.max_cnt = a_caps->jfr.max_cnt; + udma_dev->caps.jfr.depth = a_caps->jfr.depth; + udma_dev->caps.jfr.start_idx = a_caps->jfr.start_idx; + udma_dev->caps.jfc.max_cnt = a_caps->jfc.max_cnt; + udma_dev->caps.jfc.depth = a_caps->jfc.depth; + udma_dev->caps.jfc.start_idx = a_caps->jfc.start_idx; + udma_dev->caps.jetty.max_cnt = a_caps->jfs.max_cnt; + udma_dev->caps.jetty.depth = a_caps->jfs.depth; + udma_dev->caps.jetty.start_idx = a_caps->jfs.start_idx; + udma_dev->caps.jetty.next_idx = udma_dev->caps.jetty.start_idx; + udma_dev->caps.cqe_size = UDMA_CQE_SIZE; + udma_dev->caps.rc_queue_num = a_caps->rc_max_cnt; + udma_dev->caps.rc_queue_depth = a_caps->rc_que_depth; + udma_dev->caps.rc_entry_size = RC_QUEUE_ENTRY_SIZE; + udma_dev->caps.rc_dma_len = a_caps->pmem.dma_len; + udma_dev->caps.rc_dma_addr = a_caps->pmem.dma_addr; + + cal_max_2m_num(udma_dev); + + ret = udma_construct_qos_param(udma_dev); + if (ret) + return ret; + + udma_dev->caps.max_msg_len = MAX_MSG_LEN; + udma_dev->caps.jetty_in_grp = MAX_JETTY_IN_JETTY_GRP; + + if (udma_dev->caps.jetty_in_grp) { + jetty_grp_cnt = udma_dev->caps.jetty.max_cnt / udma_dev->caps.jetty_in_grp; + udma_dev->caps.jetty_grp.max_cnt = + jetty_grp_cnt < udma_dev->caps.jetty_grp.max_cnt ? + jetty_grp_cnt : udma_dev->caps.jetty_grp.max_cnt; + } + + return 0; +} + +static int udma_init_dev_param(struct udma_dev *udma_dev) +{ + struct auxiliary_device *adev = udma_dev->comdev.adev; + struct ubase_resource_space *mem_base = ubase_get_mem_base(adev); + int ret; + int i; + + udma_dev->dev = adev->dev.parent; + udma_dev->db_base = mem_base->addr_unmapped; + udma_dev->k_db_base = mem_base->addr; + udma_dev->adev_id = udma_dev->comdev.adev->id; + + ret = udma_set_hw_caps(udma_dev); + if (ret) { + dev_err(udma_dev->dev, "failed to query hw caps, ret = %d\n", ret); + return ret; + } + + ret = udma_init_tables(udma_dev); + if (ret) { + dev_err(udma_dev->dev, + "Failed to init tables, ret = %d\n", ret); + return ret; + } + + dev_set_drvdata(&adev->dev, udma_dev); + + mutex_init(&udma_dev->db_mutex); + for (i = 0; i < UDMA_DB_TYPE_NUM; i++) + INIT_LIST_HEAD(&udma_dev->db_list[i]); + + udma_init_hugepage(udma_dev); + + return 0; +} + +static void udma_uninit_dev_param(struct udma_dev *udma_dev) +{ + udma_destroy_hugepage(udma_dev); + mutex_destroy(&udma_dev->db_mutex); + dev_set_drvdata(&udma_dev->comdev.adev->dev, NULL); + udma_destroy_tables(udma_dev); +} + +static int udma_alloc_dev_tid(struct udma_dev *udma_dev) +{ + struct ummu_seg_attr seg_attr = {.token = NULL, .e_bit = UMMU_EBIT_ON}; + struct ummu_param param = {.mode = MAPT_MODE_TABLE}; + int ret; + + ret = iommu_dev_enable_feature(udma_dev->dev, IOMMU_DEV_FEAT_KSVA); + if (ret) { + dev_err(udma_dev->dev, "enable ksva failed, ret = %d.\n", ret); + return ret; + } + + ret = iommu_dev_enable_feature(udma_dev->dev, IOMMU_DEV_FEAT_SVA); + if (ret) { + dev_err(udma_dev->dev, "enable sva failed, ret = %d.\n", ret); + goto err_sva_enable_dev; + } + + udma_dev->ksva = ummu_ksva_bind_device(udma_dev->dev, ¶m); + if (!udma_dev->ksva) { + dev_err(udma_dev->dev, "ksva bind device failed.\n"); + ret = -EINVAL; + goto err_ksva_bind_device; + } + + ret = ummu_get_tid(udma_dev->dev, udma_dev->ksva, &udma_dev->tid); + if (ret) { + dev_err(udma_dev->dev, "Failed to get tid for udma device.\n"); + goto err_get_tid; + } + + ret = ummu_sva_grant_range(udma_dev->ksva, 0, UDMA_MAX_GRANT_SIZE, + UMMU_DEV_WRITE | UMMU_DEV_READ, &seg_attr); + if (ret) { + dev_err(udma_dev->dev, "Failed to sva grant range for udma device.\n"); + goto err_sva_grant_range; + } + + return ret; + +err_sva_grant_range: +err_get_tid: + ummu_ksva_unbind_device(udma_dev->ksva); +err_ksva_bind_device: + if (iommu_dev_disable_feature(udma_dev->dev, IOMMU_DEV_FEAT_SVA)) + dev_warn(udma_dev->dev, "disable sva failed.\n"); +err_sva_enable_dev: + if (iommu_dev_disable_feature(udma_dev->dev, IOMMU_DEV_FEAT_KSVA)) + dev_warn(udma_dev->dev, "disable ksva failed.\n"); + return ret; +} + +static void udma_free_dev_tid(struct udma_dev *udma_dev) +{ + struct iommu_sva *ksva = NULL; + size_t token_id; + int ret; + + ret = ummu_sva_ungrant_range(udma_dev->ksva, 0, UDMA_MAX_GRANT_SIZE, NULL); + if (ret) + dev_warn(udma_dev->dev, + "sva ungrant range for udma device failed, ret = %d.\n", + ret); + + mutex_lock(&udma_dev->ksva_mutex); + xa_for_each(&udma_dev->ksva_table, token_id, ksva) { + __xa_erase(&udma_dev->ksva_table, token_id); + ummu_ksva_unbind_device(ksva); + } + mutex_unlock(&udma_dev->ksva_mutex); + + ummu_ksva_unbind_device(udma_dev->ksva); + + ret = iommu_dev_disable_feature(udma_dev->dev, IOMMU_DEV_FEAT_SVA); + if (ret) + dev_warn(udma_dev->dev, "disable sva failed, ret = %d.\n", ret); + + ret = iommu_dev_disable_feature(udma_dev->dev, IOMMU_DEV_FEAT_KSVA); + if (ret) + dev_warn(udma_dev->dev, "disable ksva failed, ret = %d.\n", ret); +} + +static int udma_create_db_page(struct udma_dev *udev) +{ + udev->db_page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!udev->db_page) + return -ENOMEM; + + return 0; +} + +static void udma_destroy_db_page(struct udma_dev *udev) +{ + put_page(udev->db_page); + udev->db_page = NULL; +} + +static const struct udma_func_map udma_dev_func_map[] = { + {"dev param", udma_init_dev_param, udma_uninit_dev_param}, + {"cmd", udma_cmd_init, udma_cmd_cleanup}, + {"dev tid", udma_alloc_dev_tid, udma_free_dev_tid}, + {"dfx", udma_dfx_init, udma_dfx_uninit}, + {"db page", udma_create_db_page, udma_destroy_db_page}, +}; + +static void udma_destroy_dev(struct udma_dev *udev) +{ + int i; + + for (i = ARRAY_SIZE(udma_dev_func_map) - 1; i >= 0; i--) + if (udma_dev_func_map[i].uninit_func) + udma_dev_func_map[i].uninit_func(udev); + kfree(udev); +} + +static struct udma_dev *udma_create_dev(struct auxiliary_device *adev) +{ + struct udma_dev *udma_dev; + int ret, i; + + udma_dev = kzalloc((sizeof(struct udma_dev)), GFP_KERNEL); + if (udma_dev == NULL) + return NULL; + + udma_dev->comdev.adev = adev; + + for (i = 0; i < ARRAY_SIZE(udma_dev_func_map); i++) { + if (!udma_dev_func_map[i].init_func) + continue; + + ret = udma_dev_func_map[i].init_func(udma_dev); + if (ret) { + dev_err(udma_dev->dev, "Failed to init %s, ret = %d\n", + udma_dev_func_map[i].err_msg, ret); + goto err_init; + } + } + + return udma_dev; + +err_init: + for (i -= 1; i >= 0; i--) + if (udma_dev_func_map[i].uninit_func) + udma_dev_func_map[i].uninit_func(udma_dev); + + kfree(udma_dev); + return NULL; +} + +static void udma_port_handler(struct auxiliary_device *adev, bool link_up) +{ + struct udma_dev *udma_dev = get_udma_dev(adev); + struct ubcore_event ae = {}; + + ae.ub_dev = &udma_dev->ub_dev; + + if (link_up) + ae.event_type = UBCORE_EVENT_PORT_ACTIVE; + else + ae.event_type = UBCORE_EVENT_PORT_DOWN; + + ae.element.port_id = udma_dev->port_id; + dev_info(udma_dev->dev, + "udma report port event %s, matched udma dev(%s).\n", + link_up ? "ACTIVE" : "DOWN", udma_dev->dev_name); + + ubcore_dispatch_async_event(&ae); +} + +static int udma_register_event(struct auxiliary_device *adev) +{ + int ret; + + ret = udma_register_ae_event(adev); + if (ret) + return ret; + + ret = udma_register_ce_event(adev); + if (ret) + goto err_ce_register; + + ret = udma_register_crq_event(adev); + if (ret) + goto err_crq_register; + + ret = udma_register_ctrlq_event(adev); + if (ret) + goto err_ctrlq_register; + + ubase_port_register(adev, udma_port_handler); + + return 0; + +err_ctrlq_register: + udma_unregister_crq_event(adev); +err_crq_register: + udma_unregister_ce_event(adev); +err_ce_register: + udma_unregister_ae_event(adev); + + return ret; +} + +static void udma_unregister_event(struct auxiliary_device *adev) +{ + ubase_port_unregister(adev); + udma_unregister_ctrlq_event(adev); + udma_unregister_crq_event(adev); + udma_unregister_ce_event(adev); + udma_unregister_ae_event(adev); +} + +static bool udma_is_need_probe(struct auxiliary_device *adev) +{ + struct udma_dev *udma_dev; + + if (is_rmmod) { + dev_info(&adev->dev, + "udma drv is uninstalling, not allowed to create dev(%s.%u).\n", + adev->name, adev->id); + return false; + } + + udma_dev = get_udma_dev(adev); + if (udma_dev) { + dev_info(&adev->dev, + "dev(%s.%u) is exist, bypass probe.\n", + adev->name, adev->id); + return false; + } + + return true; +} + +static void udma_report_reset_event(enum ubcore_event_type event_type, + struct udma_dev *udma_dev) +{ + struct ubcore_event ae = {}; + + ae.ub_dev = &udma_dev->ub_dev; + ae.event_type = event_type; + + if (event_type == UBCORE_EVENT_ELR_ERR) + dev_info(udma_dev->dev, + "udma report reset event elr_err, matched udma dev(%s).\n", + udma_dev->dev_name); + else if (event_type == UBCORE_EVENT_ELR_DONE) + dev_info(udma_dev->dev, + "udma report reset event elr_done, matched udma dev(%s).\n", + udma_dev->dev_name); + + ubcore_dispatch_async_event(&ae); +} + +static void udma_reset_handler(struct auxiliary_device *adev, + enum ubase_reset_stage stage) +{ + switch (stage) { + case UBASE_RESET_STAGE_DOWN: + udma_reset_down(adev); + break; + case UBASE_RESET_STAGE_UNINIT: + udma_reset_uninit(adev); + break; + case UBASE_RESET_STAGE_INIT: + udma_reset_init(adev); + break; + default: + break; + } +} + +static int udma_init_eid_table(struct udma_dev *udma_dev) +{ + int ret; + + ret = udma_query_eid_from_ctrl_cpu(udma_dev); + if (ret) + dev_err(udma_dev->dev, "query eid info failed, ret = %d.\n", ret); + + return ret; +} + +static int udma_init_dev(struct auxiliary_device *adev) +{ + struct udma_dev *udma_dev; + int ret; + + mutex_lock(&udma_reset_mutex); + dev_info(&adev->dev, "udma init dev called, matched aux dev(%s.%u).\n", + adev->name, adev->id); + if (!udma_is_need_probe(adev)) { + mutex_unlock(&udma_reset_mutex); + return 0; + } + + udma_dev = udma_create_dev(adev); + if (!udma_dev) + goto err_create; + + ret = udma_register_event(adev); + if (ret) + goto err_event_register; + + ret = udma_register_activate_workqueue(udma_dev); + if (ret) { + dev_err(udma_dev->dev, "UDMA activate workqueue failed.\n"); + goto err_register_act_init; + } + + ret = udma_set_ubcore_dev(udma_dev); + if (ret) { + dev_err(udma_dev->dev, "failed to set ubcore dev, ret is %d.\n", ret); + goto err_set_ubcore_dev; + } + + ret = udma_init_eid_table(udma_dev); + if (ret) { + dev_err(udma_dev->dev, "init eid table failed.\n"); + goto err_init_eid; + } + udma_register_debugfs(udma_dev); + udma_dev->status = UDMA_NORMAL; + mutex_unlock(&udma_reset_mutex); + dev_info(udma_dev->dev, "init udma successfully.\n"); + + return 0; + +err_init_eid: + udma_unset_ubcore_dev(udma_dev); +err_set_ubcore_dev: + udma_unregister_activate_workqueue(udma_dev); +err_register_act_init: + udma_unregister_event(adev); +err_event_register: + udma_destroy_dev(udma_dev); +err_create: + mutex_unlock(&udma_reset_mutex); + + return -EINVAL; +} + +static void check_and_wait_flush_done(struct udma_dev *udma_dev) +{ +#define WAIT_MAX_TIMES 15 + uint32_t wait_times = 0; + + while (true) { + if (udma_dev->disable_ue_rx_count == 1) + break; + + if (wait_times > WAIT_MAX_TIMES) { + dev_warn(udma_dev->dev, "wait flush done timeout.\n"); + break; + } + msleep(1 << wait_times); + wait_times++; + } +} + +void udma_reset_down(struct auxiliary_device *adev) +{ + struct udma_dev *udma_dev; + + mutex_lock(&udma_reset_mutex); + udma_dev = get_udma_dev(adev); + if (!udma_dev) { + mutex_unlock(&udma_reset_mutex); + dev_info(&adev->dev, "udma device is not exist.\n"); + return; + } + + if (udma_dev->status != UDMA_NORMAL) { + mutex_unlock(&udma_reset_mutex); + dev_info(&adev->dev, "udma device status(%u).\n", udma_dev->status); + return; + } + + ubcore_stop_requests(&udma_dev->ub_dev); + udma_report_reset_event(UBCORE_EVENT_ELR_ERR, udma_dev); + udma_dev->status = UDMA_SUSPEND; + mutex_unlock(&udma_reset_mutex); +} + +void udma_reset_uninit(struct auxiliary_device *adev) +{ + struct udma_dev *udma_dev; + + mutex_lock(&udma_reset_mutex); + udma_dev = get_udma_dev(adev); + if (!udma_dev) { + dev_info(&adev->dev, "udma device is not exist.\n"); + mutex_unlock(&udma_reset_mutex); + return; + } + + if (udma_dev->status != UDMA_SUSPEND) { + dev_info(&adev->dev, "udma device status(%u).\n", udma_dev->status); + mutex_unlock(&udma_reset_mutex); + return; + } + + if (udma_close_ue_rx(udma_dev, false, false, true, 0)) { + mutex_unlock(&udma_reset_mutex); + dev_err(&adev->dev, "udma close ue rx failed in reset process.\n"); + return; + } + + /* Event should unregister before unset ubcore dev. */ + udma_unregister_event(adev); + udma_unset_ubcore_dev(udma_dev); + udma_unregister_debugfs(udma_dev); + udma_unregister_activate_workqueue(udma_dev); + udma_open_ue_rx(udma_dev, false, false, true, 0); + udma_destroy_dev(udma_dev); + mutex_unlock(&udma_reset_mutex); +} + +void udma_reset_init(struct auxiliary_device *adev) +{ + udma_init_dev(adev); +} + +int udma_probe(struct auxiliary_device *adev, + const struct auxiliary_device_id *id) +{ + if (udma_init_dev(adev)) + return -EINVAL; + + ubase_reset_register(adev, udma_reset_handler); + return 0; +} + +void udma_remove(struct auxiliary_device *adev) +{ + struct udma_dev *udma_dev; + + ubase_reset_unregister(adev); + mutex_lock(&udma_reset_mutex); + udma_dev = get_udma_dev(adev); + if (!udma_dev) { + mutex_unlock(&udma_reset_mutex); + dev_info(&adev->dev, "udma device is not exist.\n"); + return; + } + + ubcore_stop_requests(&udma_dev->ub_dev); + if (udma_close_ue_rx(udma_dev, false, false, false, 0)) { + mutex_unlock(&udma_reset_mutex); + dev_err(&adev->dev, "udma close ue rx failed in remove process.\n"); + return; + } + + udma_dev->status = UDMA_SUSPEND; + udma_report_reset_event(UBCORE_EVENT_ELR_ERR, udma_dev); + + /* Event should unregister before unset ubcore dev. */ + udma_unregister_event(adev); + udma_unset_ubcore_dev(udma_dev); + udma_unregister_debugfs(udma_dev); + udma_unregister_activate_workqueue(udma_dev); + check_and_wait_flush_done(udma_dev); + (void)ubase_activate_dev(adev); + udma_destroy_dev(udma_dev); + mutex_unlock(&udma_reset_mutex); + dev_info(&adev->dev, "udma device remove success.\n"); +} + +static struct auxiliary_driver udma_drv = { + .name = "udma", + .probe = udma_probe, + .remove = udma_remove, + .id_table = udma_id_table, +}; + +static int __init udma_init(void) +{ + int ret; + + udma_init_debugfs(); + ret = auxiliary_driver_register(&udma_drv); + if (ret) { + pr_err("failed to register auxiliary_driver\n"); + udma_uninit_debugfs(); + } + + return ret; +} + +static void __exit udma_exit(void) +{ + is_rmmod = true; + auxiliary_driver_unregister(&udma_drv); + udma_uninit_debugfs(); +} + +module_init(udma_init); +module_exit(udma_exit); +MODULE_LICENSE("GPL"); + +module_param(cqe_mode, bool, 0444); +MODULE_PARM_DESC(cqe_mode, "Set cqe reporting mode, default: 1 (0:BY_COUNT, 1:BY_CI_PI_GAP)"); + +module_param(jfr_sleep_time, uint, 0444); +MODULE_PARM_DESC(jfr_sleep_time, "Set the destroy jfr sleep time, default: 1000 us.\n"); + +module_param(jfc_arm_mode, uint, 0444); +MODULE_PARM_DESC(jfc_arm_mode, + "Set the ARM mode of the JFC, default: 0(0:Always ARM, other: NO ARM."); + +module_param(dump_aux_info, bool, 0644); +MODULE_PARM_DESC(dump_aux_info, + "Set whether dump aux info, default: false(false:not print, true:print)"); diff --git a/drivers/ub/urma/hw/udma/udma_rct.c b/drivers/ub/urma/hw/udma/udma_rct.c new file mode 100644 index 0000000000000000000000000000000000000000..ee11d3ef3ee9b828197cc4413ff7acfc2cb29031 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_rct.c @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt + +#include +#include "udma_cmd.h" +#include "udma_rct.h" + +static int udma_create_rc_queue_ctx(struct udma_dev *dev, struct udma_rc_queue *rcq) +{ + struct ubase_mbx_attr attr = {}; + struct udma_rc_ctx ctx = {}; + + ctx.type = RC_TYPE; + ctx.state = RC_READY_STATE; + ctx.rce_token_id_l = dev->tid & (uint32_t)RCE_TOKEN_ID_L_MASK; + ctx.rce_token_id_h = dev->tid >> RCE_TOKEN_ID_H_OFFSET; + ctx.rce_base_addr_l = (rcq->buf.addr >> RCE_ADDR_L_OFFSET) & + (uint32_t)RCE_ADDR_L_MASK; + ctx.rce_base_addr_h = rcq->buf.addr >> RCE_ADDR_H_OFFSET; + ctx.rce_shift = ilog2(roundup_pow_of_two(rcq->buf.entry_cnt)); + ctx.avail_sgmt_ost = RC_AVAIL_SGMT_OST; + + attr.tag = rcq->id; + attr.op = UDMA_CMD_CREATE_RC_CONTEXT; + + return post_mailbox_update_ctx(dev, &ctx, sizeof(ctx), &attr); +} + +static int udma_destroy_rc_queue_ctx(struct udma_dev *dev, struct udma_rc_queue *rcq) +{ + struct ubase_mbx_attr mbox_attr = {}; + struct ubase_cmd_mailbox *mailbox; + int ret; + + mailbox = udma_alloc_cmd_mailbox(dev); + if (!mailbox) { + dev_err(dev->dev, "failed to alloc mailbox for rc queue.\n"); + return -ENOMEM; + } + + mbox_attr.tag = rcq->id; + mbox_attr.op = UDMA_CMD_DESTROY_RC_CONTEXT; + ret = udma_post_mbox(dev, mailbox, &mbox_attr); + if (ret) + dev_err(dev->dev, "failed to destroy rc queue ctx, ret = %d.\n", ret); + + udma_free_cmd_mailbox(dev, mailbox); + + return ret; +} + +static int udma_alloc_rct_buffer(struct udma_dev *dev, struct ubcore_device_cfg *cfg, + struct udma_rc_queue *rcq) +{ + uint32_t rct_buffer_size = dev->caps.rc_entry_size * cfg->rc_cfg.depth; + uint32_t buf_num_per_hugepage; + + rcq->buf.entry_size = dev->caps.rc_entry_size; + rcq->buf.entry_cnt = cfg->rc_cfg.depth; + if (ubase_adev_prealloc_supported(dev->comdev.adev)) { + rct_buffer_size = ALIGN(rct_buffer_size, PAGE_SIZE); + if (rct_buffer_size > UDMA_HUGEPAGE_SIZE) { + rcq->buf.addr = dev->caps.rc_dma_addr + rcq->id * rct_buffer_size; + } else { + buf_num_per_hugepage = UDMA_HUGEPAGE_SIZE / rct_buffer_size; + rcq->buf.addr = dev->caps.rc_dma_addr + + rcq->id / buf_num_per_hugepage * UDMA_HUGEPAGE_SIZE + + rcq->id % buf_num_per_hugepage * rct_buffer_size; + } + } else { + rcq->buf.kva_or_slot = udma_alloc_iova(dev, rct_buffer_size, &rcq->buf.addr); + if (!rcq->buf.kva_or_slot) { + dev_err(dev->dev, "failed to alloc rct buffer.\n"); + return -ENOMEM; + } + } + + return 0; +} + +static void udma_free_rct_buffer(struct udma_dev *dev, struct udma_rc_queue *rcq) +{ + uint32_t rct_buffer_size = rcq->buf.entry_size * rcq->buf.entry_cnt; + + if (!ubase_adev_prealloc_supported(dev->comdev.adev)) { + udma_free_iova(dev, rct_buffer_size, rcq->buf.kva_or_slot, rcq->buf.addr); + rcq->buf.kva_or_slot = NULL; + rcq->buf.addr = 0; + } +} + +static int udma_alloc_rc_queue(struct udma_dev *dev, + struct ubcore_device_cfg *cfg, int rc_queue_id) +{ + struct udma_rc_queue *rcq; + int ret; + + rcq = kzalloc(sizeof(struct udma_rc_queue), GFP_KERNEL); + if (!rcq) + return -ENOMEM; + rcq->id = rc_queue_id; + + ret = udma_alloc_rct_buffer(dev, cfg, rcq); + if (ret) + goto err_alloc_rct_buffer; + + ret = udma_create_rc_queue_ctx(dev, rcq); + if (ret) { + dev_err(dev->dev, + "failed to create rc queue ctx, rcq id %u, ret = %d.\n", + rcq->id, ret); + goto err_create_rcq_ctx; + } + + ret = xa_err(xa_store(&dev->rc_table.xa, rcq->id, rcq, GFP_KERNEL)); + if (ret) { + dev_err(dev->dev, + "failed to stored rcq id to rc table, rcq id %d.\n", + rc_queue_id); + goto err_store_rcq_id; + } + + if (dfx_switch) + udma_dfx_store_id(dev, &dev->dfx_info->rc, rcq->id, "rc"); + + return ret; + +err_store_rcq_id: + if (udma_destroy_rc_queue_ctx(dev, rcq)) + dev_err(dev->dev, + "udma destroy rc queue ctx failed when alloc rc queue.\n"); +err_create_rcq_ctx: + udma_free_rct_buffer(dev, rcq); +err_alloc_rct_buffer: + kfree(rcq); + + return ret; +} + +void udma_free_rc_queue(struct udma_dev *dev, int rc_queue_id) +{ + struct udma_rc_queue *rcq; + int ret; + + rcq = (struct udma_rc_queue *)xa_load(&dev->rc_table.xa, rc_queue_id); + if (!rcq) { + dev_warn(dev->dev, + "failed to find rcq, id = %d.\n", rc_queue_id); + return; + } + + xa_erase(&dev->rc_table.xa, rc_queue_id); + ret = udma_destroy_rc_queue_ctx(dev, rcq); + if (ret) + dev_err(dev->dev, + "udma destroy rc queue ctx failed, ret = %d.\n", ret); + + if (dfx_switch) + udma_dfx_delete_id(dev, &dev->dfx_info->rc, rc_queue_id); + + udma_free_rct_buffer(dev, rcq); + kfree(rcq); +} + +static int udma_config_rc_table(struct udma_dev *dev, struct ubcore_device_cfg *cfg) +{ + uint32_t rc_ctx_num = cfg->rc_cfg.rc_cnt; + int ret = 0; + int i; + + for (i = 0; i < rc_ctx_num; i++) { + ret = udma_alloc_rc_queue(dev, cfg, i); + if (ret) { + dev_err(dev->dev, "failed to alloc rc queue.\n"); + goto err_alloc_rc_queue; + } + } + dev->rc_table.ida_table.min = 0; + dev->rc_table.ida_table.max = rc_ctx_num; + + return ret; + +err_alloc_rc_queue: + for (i -= 1; i >= 0; i--) + udma_free_rc_queue(dev, i); + + return ret; +} + +static int check_and_config_rc_table(struct udma_dev *dev, struct ubcore_device_cfg *cfg) +{ + int ret = 0; + + if (!cfg->mask.bs.rc_cnt && !cfg->mask.bs.rc_depth) + return 0; + + if (!cfg->mask.bs.rc_cnt || !cfg->mask.bs.rc_depth) { + dev_err(dev->dev, "Invalid rc mask, mask = %u.\n", cfg->mask.value); + return -EINVAL; + } + + if (!cfg->rc_cfg.rc_cnt || !cfg->rc_cfg.depth || + cfg->rc_cfg.rc_cnt > dev->caps.rc_queue_num || + cfg->rc_cfg.rc_cnt <= dev->caps.ack_queue_num) { + dev_err(dev->dev, + "Invalid rc param, rc cnt = %u, rc depth = %u, rc num = %u, ack queue num = %u.\n", + cfg->rc_cfg.rc_cnt, cfg->rc_cfg.depth, + dev->caps.rc_queue_num, dev->caps.ack_queue_num); + return -EINVAL; + } + + if (!test_and_set_bit_lock(RCT_INIT_FLAG, &dev->caps.init_flag)) + ret = udma_config_rc_table(dev, cfg); + + return ret; +} + +int udma_config_device(struct ubcore_device *ubcore_dev, + struct ubcore_device_cfg *cfg) +{ + struct udma_dev *dev = to_udma_dev(ubcore_dev); + int ret; + + if ((cfg->mask.bs.reserved_jetty_id_min && cfg->reserved_jetty_id_min != 0) || + (cfg->mask.bs.reserved_jetty_id_max && cfg->reserved_jetty_id_max != + dev->caps.public_jetty.max_cnt - 1)) { + dev_err(dev->dev, "public jetty range must 0-%u.\n", + dev->caps.public_jetty.max_cnt - 1); + return -EINVAL; + } + + ret = check_and_config_rc_table(dev, cfg); + if (ret) + dev_err(dev->dev, "failed to check device cfg, ret = %d.\n", ret); + + return ret; +} diff --git a/drivers/ub/urma/hw/udma/udma_rct.h b/drivers/ub/urma/hw/udma/udma_rct.h new file mode 100644 index 0000000000000000000000000000000000000000..fc1e47d95043b632f5e27854cfa37044581c59b2 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_rct.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_RCT_H__ +#define __UDMA_RCT_H__ + +#include "udma_common.h" + +#define RC_TYPE 2U +#define RC_READY_STATE 1U +#define RC_AVAIL_SGMT_OST 512U + +#define RCE_TOKEN_ID_L_MASK GENMASK(11, 0) +#define RCE_TOKEN_ID_H_OFFSET 12U +#define RCE_ADDR_L_OFFSET 12U +#define RCE_ADDR_L_MASK GENMASK(19, 0) +#define RCE_ADDR_H_OFFSET 32U + +struct udma_rc_queue { + uint32_t id; + struct udma_buf buf; +}; + +struct udma_rc_ctx { + /* DW0 */ + uint32_t rsv0 : 5; + uint32_t type : 3; + uint32_t rce_shift : 4; + uint32_t rsv1 : 4; + uint32_t state : 3; + uint32_t rsv2 : 1; + uint32_t rce_token_id_l : 12; + /* DW1 */ + uint32_t rce_token_id_h : 8; + uint32_t rsv3 : 4; + uint32_t rce_base_addr_l : 20; + /* DW2 */ + uint32_t rce_base_addr_h; + /* DW3~DW31 */ + uint32_t rsv4[28]; + uint32_t avail_sgmt_ost : 10; + uint32_t rsv5 : 22; + /* DW32~DW63 */ + uint32_t rsv6[32]; +}; + +struct udma_vir_cap { + uint8_t ue_idx; + uint8_t virtualization : 1; + uint8_t rsv : 7; +}; + +int udma_config_device(struct ubcore_device *ubcore_dev, + struct ubcore_device_cfg *cfg); +void udma_free_rc_queue(struct udma_dev *dev, int rc_queue_id); + +#endif /* __UDMA_RCT_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_segment.c b/drivers/ub/urma/hw/udma/udma_segment.c new file mode 100644 index 0000000000000000000000000000000000000000..7bb52ed4d5a164c04700bbd7f05c5c8676487f1e --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_segment.c @@ -0,0 +1,367 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt + +#include +#include +#include "udma_common.h" +#include "udma_cmd.h" +#include "udma_eid.h" +#include "udma_tid.h" +#include "udma_segment.h" + +static int udma_pin_segment(struct ubcore_device *ub_dev, struct ubcore_seg_cfg *cfg, + struct udma_segment *seg) +{ + struct udma_dev *udma_dev = to_udma_dev(ub_dev); + struct udma_umem_param param; + int ret = 0; + + if (cfg->flag.bs.non_pin) + return 0; + + param.ub_dev = ub_dev; + param.va = cfg->va; + param.len = cfg->len; + + param.flag.bs.writable = + !!(cfg->flag.bs.access & UBCORE_ACCESS_WRITE); + param.flag.bs.non_pin = cfg->flag.bs.non_pin; + param.is_kernel = seg->kernel_mode; + + seg->umem = udma_umem_get(¶m); + if (IS_ERR(seg->umem)) { + ret = PTR_ERR(seg->umem); + dev_err(udma_dev->dev, + "failed to get segment umem, ret = %d.\n", ret); + } + + return ret; +} + +static void udma_unpin_seg(struct udma_segment *seg) +{ + if (!seg->core_tseg.seg.attr.bs.non_pin) + udma_umem_release(seg->umem, seg->kernel_mode); +} + +static int udma_check_seg_cfg(struct udma_dev *udma_dev, struct ubcore_seg_cfg *cfg) +{ + if (!cfg->token_id || cfg->flag.bs.access >= UDMA_SEGMENT_ACCESS_GUARD || + cfg->flag.bs.token_policy >= UBCORE_TOKEN_SIGNED) { + dev_err(udma_dev->dev, + "error segment input, access = %d, token_policy = %d, or null key_id.\n", + cfg->flag.bs.access, cfg->flag.bs.token_policy); + return -EINVAL; + } + + return 0; +} + +static void udma_init_seg_cfg(struct udma_segment *seg, struct ubcore_seg_cfg *cfg) +{ + seg->core_tseg.token_id = cfg->token_id; + seg->core_tseg.seg.token_id = cfg->token_id->token_id; + seg->core_tseg.seg.attr.value = cfg->flag.value; + seg->token_value = cfg->token_value.token; + seg->token_value_valid = cfg->flag.bs.token_policy != UBCORE_TOKEN_NONE; +} + +static int udma_u_get_seg_perm(struct ubcore_seg_cfg *cfg) +{ + bool local_only_flag = cfg->flag.bs.access & UBCORE_ACCESS_LOCAL_ONLY; + bool atomic_flag = cfg->flag.bs.access & UBCORE_ACCESS_ATOMIC; + bool write_flag = cfg->flag.bs.access & UBCORE_ACCESS_WRITE; + bool read_flag = cfg->flag.bs.access & UBCORE_ACCESS_READ; + + /* After setting ACCESS_LOCAL, other operations cannot be configured. */ + if (local_only_flag && !atomic_flag && !write_flag && !read_flag) + return UMMU_DEV_ATOMIC | UMMU_DEV_WRITE | UMMU_DEV_READ; + + /* Atomic require additional configuration of write and read. */ + if (!local_only_flag && atomic_flag && write_flag && read_flag) + return UMMU_DEV_ATOMIC | UMMU_DEV_WRITE | UMMU_DEV_READ; + + /* Write require additional configuration of read. */ + if (!local_only_flag && !atomic_flag && write_flag && read_flag) + return UMMU_DEV_WRITE | UMMU_DEV_READ; + + if (!local_only_flag && !atomic_flag && !write_flag && read_flag) + return UMMU_DEV_READ; + + /* All other configurations are illegal. */ + return 0; +} + +static int udma_sva_grant(struct ubcore_seg_cfg *cfg, struct iommu_sva *ksva) +{ +#define UDMA_TOKEN_VALUE_INPUT 0 + struct ummu_seg_attr seg_attr = {.token = NULL, .e_bit = UMMU_EBIT_ON}; + struct ummu_token_info token_info; + int perm; + int ret; + + perm = udma_u_get_seg_perm(cfg); + seg_attr.e_bit = (enum ummu_ebit_state)cfg->flag.bs.access & + UBCORE_ACCESS_LOCAL_ONLY; + + if (cfg->flag.bs.token_policy == UBCORE_TOKEN_NONE) { + return ummu_sva_grant_range(ksva, (void *)(uintptr_t)cfg->va, cfg->len, + perm, (void *)&seg_attr); + } else { + seg_attr.token = &token_info; + token_info.input = UDMA_TOKEN_VALUE_INPUT; + token_info.tokenVal = cfg->token_value.token; + ret = ummu_sva_grant_range(ksva, (void *)(uintptr_t)cfg->va, cfg->len, + perm, (void *)&seg_attr); + token_info.tokenVal = 0; + + return ret; + } +} + +static int udma_set_seg_eid(struct udma_dev *udma_dev, uint32_t eid_index, + union ubcore_eid *eid) +{ + struct ubase_mbx_attr mbox_attr = {}; + struct ubase_cmd_mailbox *mailbox; + struct udma_seid_upi *seid_upi; + int ret; + + if (udma_dev->is_ue) { + dev_info(udma_dev->dev, + "The ue does not support the delivery of seid(%u) mailbox.\n", + eid_index); + return 0; + } + + mailbox = udma_alloc_cmd_mailbox(udma_dev); + if (!mailbox) { + dev_err(udma_dev->dev, + "failed to alloc mailbox for get tp seid.\n"); + return -ENOMEM; + } + + mbox_attr.tag = eid_index; + mbox_attr.op = UDMA_CMD_READ_SEID_UPI; + ret = udma_post_mbox(udma_dev, mailbox, &mbox_attr); + if (ret) { + dev_err(udma_dev->dev, + "send tp eid table mailbox cmd failed, ret is %d.\n", ret); + } else { + seid_upi = (struct udma_seid_upi *)mailbox->buf; + *eid = seid_upi->seid; + } + + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +static void udma_dfx_store_seg(struct udma_dev *udma_dev, + struct ubcore_seg_cfg *cfg) +{ + struct udma_dfx_seg *seg; + int ret; + + seg = (struct udma_dfx_seg *)xa_load(&udma_dev->dfx_info->seg.table, + cfg->token_id->token_id); + if (seg) { + dev_warn(udma_dev->dev, "segment already exists in DFX.\n"); + return; + } + + seg = kzalloc(sizeof(*seg), GFP_KERNEL); + if (!seg) + return; + + seg->id = cfg->token_id->token_id; + seg->ubva.va = cfg->va; + seg->len = cfg->len; + seg->token_value = cfg->token_value; + if (udma_set_seg_eid(udma_dev, cfg->eid_index, &seg->ubva.eid)) { + kfree(seg); + return; + } + + write_lock(&udma_dev->dfx_info->seg.rwlock); + ret = xa_err(xa_store(&udma_dev->dfx_info->seg.table, cfg->token_id->token_id, + seg, GFP_KERNEL)); + if (ret) { + write_unlock(&udma_dev->dfx_info->seg.rwlock); + dev_err(udma_dev->dev, "store segment to table failed in DFX.\n"); + kfree(seg); + return; + } + + ++udma_dev->dfx_info->seg.cnt; + write_unlock(&udma_dev->dfx_info->seg.rwlock); +} + +static void udma_dfx_delete_seg(struct udma_dev *udma_dev, uint32_t token_id, + uint64_t va) +{ + struct udma_dfx_seg *seg; + + write_lock(&udma_dev->dfx_info->seg.rwlock); + seg = (struct udma_dfx_seg *)xa_load(&udma_dev->dfx_info->seg.table, + token_id); + if (seg && seg->id == token_id && seg->ubva.va == va) { + xa_erase(&udma_dev->dfx_info->seg.table, token_id); + seg->token_value.token = 0; + kfree(seg); + --udma_dev->dfx_info->seg.cnt; + } + write_unlock(&udma_dev->dfx_info->seg.rwlock); +} + +struct ubcore_target_seg *udma_register_seg(struct ubcore_device *ub_dev, + struct ubcore_seg_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(ub_dev); + struct udma_tid *udma_tid; + struct udma_segment *seg; + struct iommu_sva *ksva; + int ret = 0; + + ret = udma_check_seg_cfg(udma_dev, cfg); + if (ret) + return NULL; + + seg = kzalloc(sizeof(*seg), GFP_KERNEL); + if (!seg) + return NULL; + + seg->kernel_mode = udata == NULL; + udma_init_seg_cfg(seg, cfg); + + ret = udma_pin_segment(ub_dev, cfg, seg); + if (ret) { + dev_err(udma_dev->dev, "pin segment failed, ret = %d.\n", ret); + goto err_pin_seg; + } + + if (udata) + return &seg->core_tseg; + + udma_tid = to_udma_tid(seg->core_tseg.token_id); + + mutex_lock(&udma_dev->ksva_mutex); + ksva = (struct iommu_sva *)xa_load(&udma_dev->ksva_table, udma_tid->tid); + if (!ksva) { + dev_err(udma_dev->dev, + "unable to get ksva while register segment, token maybe is free.\n"); + goto err_load_ksva; + } + + ret = udma_sva_grant(cfg, ksva); + if (ret) { + dev_err(udma_dev->dev, + "ksva grant failed token policy %d, access %d, ret = %d.\n", + cfg->flag.bs.token_policy, cfg->flag.bs.access, ret); + goto err_load_ksva; + } + mutex_unlock(&udma_dev->ksva_mutex); + + if (dfx_switch) + udma_dfx_store_seg(udma_dev, cfg); + + return &seg->core_tseg; + +err_load_ksva: + mutex_unlock(&udma_dev->ksva_mutex); + udma_unpin_seg(seg); +err_pin_seg: + kfree(seg); + return NULL; +} + +int udma_unregister_seg(struct ubcore_target_seg *ubcore_seg) +{ + struct udma_tid *udma_tid = to_udma_tid(ubcore_seg->token_id); + struct udma_dev *udma_dev = to_udma_dev(ubcore_seg->ub_dev); + struct udma_segment *seg = to_udma_seg(ubcore_seg); + struct ummu_token_info token_info; + struct iommu_sva *ksva; + int ret; + + if (!seg->kernel_mode) + goto common_process; + + mutex_lock(&udma_dev->ksva_mutex); + ksva = (struct iommu_sva *)xa_load(&udma_dev->ksva_table, udma_tid->tid); + + if (!ksva) { + dev_warn(udma_dev->dev, + "unable to get ksva while unregister segment, token maybe is free.\n"); + } else { + if (seg->token_value_valid) { + token_info.tokenVal = seg->token_value; + ret = ummu_sva_ungrant_range(ksva, + (void *)(uintptr_t)ubcore_seg->seg.ubva.va, + ubcore_seg->seg.len, &token_info); + token_info.tokenVal = 0; + } else { + ret = ummu_sva_ungrant_range(ksva, + (void *)(uintptr_t)ubcore_seg->seg.ubva.va, + ubcore_seg->seg.len, NULL); + } + if (ret) { + mutex_unlock(&udma_dev->ksva_mutex); + dev_err(udma_dev->dev, "unregister segment failed, ret = %d.\n", ret); + return ret; + } + } + mutex_unlock(&udma_dev->ksva_mutex); + + if (dfx_switch) + udma_dfx_delete_seg(udma_dev, ubcore_seg->token_id->token_id, + ubcore_seg->seg.ubva.va); + +common_process: + udma_unpin_seg(seg); + seg->token_value = 0; + kfree(seg); + + return 0; +} + +struct ubcore_target_seg *udma_import_seg(struct ubcore_device *dev, + struct ubcore_target_seg_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_segment *seg; + + if (cfg->seg.attr.bs.token_policy > UBCORE_TOKEN_PLAIN_TEXT) { + dev_err(udma_dev->dev, "invalid token policy = %d.\n", + cfg->seg.attr.bs.token_policy); + return NULL; + } + + seg = kzalloc(sizeof(*seg), GFP_KERNEL); + if (!seg) + return NULL; + + if (cfg->seg.attr.bs.token_policy != UBCORE_TOKEN_NONE) { + seg->token_value = cfg->token_value.token; + seg->token_value_valid = true; + } + + seg->tid = cfg->seg.token_id >> UDMA_TID_SHIFT; + + return &seg->core_tseg; +} + +int udma_unimport_seg(struct ubcore_target_seg *tseg) +{ + struct udma_segment *seg; + + seg = to_udma_seg(tseg); + seg->token_value = 0; + kfree(seg); + + return 0; +} diff --git a/drivers/ub/urma/hw/udma/udma_segment.h b/drivers/ub/urma/hw/udma/udma_segment.h new file mode 100644 index 0000000000000000000000000000000000000000..7dba1fbf385efd0be9ac757bff066d2787ede7aa --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_segment.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_SEGMENT_H__ +#define __UDMA_SEGMENT_H__ + +#include "udma_dev.h" + +struct udma_segment { + struct ubcore_target_seg core_tseg; + struct ubcore_umem *umem; + uint32_t token_value; + bool token_value_valid; + bool kernel_mode; + uint32_t tid; +}; + +static inline struct udma_segment *to_udma_seg(struct ubcore_target_seg *seg) +{ + return container_of(seg, struct udma_segment, core_tseg); +} + +struct ubcore_target_seg *udma_register_seg(struct ubcore_device *ub_dev, + struct ubcore_seg_cfg *cfg, + struct ubcore_udata *udata); +int udma_unregister_seg(struct ubcore_target_seg *seg); +struct ubcore_target_seg *udma_import_seg(struct ubcore_device *dev, + struct ubcore_target_seg_cfg *cfg, + struct ubcore_udata *udata); +int udma_unimport_seg(struct ubcore_target_seg *tseg); + +#endif /* __UDMA_SEGMENT_H__ */ diff --git a/drivers/ub/urma/hw/udma/udma_tid.c b/drivers/ub/urma/hw/udma/udma_tid.c new file mode 100644 index 0000000000000000000000000000000000000000..c5b5b9037162abc5e464fefa493648b6906f8a71 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_tid.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#define dev_fmt(fmt) "UDMA: " fmt + +#include +#include "udma_tid.h" + +static int udma_get_key_id_from_user(struct udma_dev *udma_dev, + struct ubcore_udata *udata, + struct udma_tid *udma_tid) +{ + unsigned long byte; + uint32_t tid; + + if (!udata->udrv_data || !udata->udrv_data->in_addr) { + dev_err(udma_dev->dev, "udrv_data or in_addr is null.\n"); + return -EINVAL; + } + + byte = copy_from_user(&tid, (void *)(uintptr_t)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (uint32_t)sizeof(tid))); + if (byte) { + dev_err(udma_dev->dev, "get user data failed, byte = %lu.\n", byte); + return -EFAULT; + } + + udma_tid->core_key_id.token_id = tid; + udma_tid->tid = tid >> UDMA_TID_SHIFT; + + return 0; +} + +static int udma_alloc_k_tid(struct udma_dev *udma_dev, + struct udma_tid *udma_tid) +{ + struct ummu_param param = {.mode = MAPT_MODE_TABLE}; + struct iommu_sva *ksva; + uint32_t tid; + int ret; + + ksva = ummu_ksva_bind_device(udma_dev->dev, ¶m); + if (!ksva) { + dev_err(udma_dev->dev, "ksva bind device failed.\n"); + return -ENOMEM; + } + + ret = ummu_get_tid(udma_dev->dev, ksva, &tid); + if (ret) { + dev_err(udma_dev->dev, "get tid from ummu failed, ret = %d.\n", ret); + goto err_get_tid; + } + + if (tid > UDMA_MAX_TID) { + dev_err(udma_dev->dev, "tid is overflow.\n"); + ret = -EINVAL; + goto err_get_tid; + } + + mutex_lock(&udma_dev->ksva_mutex); + ret = xa_err(__xa_store(&udma_dev->ksva_table, tid, ksva, GFP_KERNEL)); + mutex_unlock(&udma_dev->ksva_mutex); + if (ret) { + dev_err(udma_dev->dev, "save ksva failed, ret = %d.\n", ret); + goto err_get_tid; + } + + udma_tid->core_key_id.token_id = tid << UDMA_TID_SHIFT; + udma_tid->tid = tid; + udma_tid->kernel_mode = true; + + return ret; + +err_get_tid: + ummu_ksva_unbind_device(ksva); + + return ret; +} + +struct ubcore_token_id *udma_alloc_tid(struct ubcore_device *ub_dev, + union ubcore_token_id_flag flag, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(ub_dev); + struct udma_tid *udma_tid; + int ret; + + udma_tid = kzalloc(sizeof(*udma_tid), GFP_KERNEL); + if (!udma_tid) + return NULL; + + if (udata) { + ret = udma_get_key_id_from_user(udma_dev, udata, udma_tid); + if (ret) { + dev_err(udma_dev->dev, "get user key id failed, ret = %d.\n", ret); + goto err_get_key_id; + } + return &udma_tid->core_key_id; + } + + if (udma_alloc_k_tid(udma_dev, udma_tid)) + goto err_get_key_id; + + return &udma_tid->core_key_id; + +err_get_key_id: + kfree(udma_tid); + return NULL; +} + +int udma_free_tid(struct ubcore_token_id *token_id) +{ + struct udma_dev *udma_dev = to_udma_dev(token_id->ub_dev); + struct udma_tid *udma_tid = to_udma_tid(token_id); + struct iommu_sva *ksva; + int ret; + + ret = ummu_core_invalidate_cfg_table(udma_tid->tid); + if (ret) + dev_err(udma_dev->dev, "invalidate cfg_table failed, ret=%d.\n", ret); + + if (!udma_tid->kernel_mode) + goto out; + + mutex_lock(&udma_dev->ksva_mutex); + ksva = (struct iommu_sva *)xa_load(&udma_dev->ksva_table, udma_tid->tid); + if (!ksva) { + mutex_unlock(&udma_dev->ksva_mutex); + dev_warn(udma_dev->dev, + "unable to get ksva while free tid, token maybe is free.\n"); + goto out; + } + ummu_ksva_unbind_device(ksva); + __xa_erase(&udma_dev->ksva_table, udma_tid->tid); + mutex_unlock(&udma_dev->ksva_mutex); + +out: + kfree(udma_tid); + + return 0; +} diff --git a/drivers/ub/urma/hw/udma/udma_tid.h b/drivers/ub/urma/hw/udma/udma_tid.h new file mode 100644 index 0000000000000000000000000000000000000000..72bf436fc23f619bd799cb2ef11b022de84f0ab9 --- /dev/null +++ b/drivers/ub/urma/hw/udma/udma_tid.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef __UDMA_TID_H__ +#define __UDMA_TID_H__ + +#include "udma_dev.h" + +struct udma_tid { + struct ubcore_token_id core_key_id; + bool kernel_mode; + uint32_t tid; +}; + +static inline struct udma_tid *to_udma_tid(struct ubcore_token_id *token_id) +{ + return container_of(token_id, struct udma_tid, core_key_id); +} + +struct ubcore_token_id *udma_alloc_tid(struct ubcore_device *dev, + union ubcore_token_id_flag flag, + struct ubcore_udata *udata); +int udma_free_tid(struct ubcore_token_id *token_id); + +#endif /* __UDMA_TID_H__ */ diff --git a/include/linux/fwctl.h b/include/linux/fwctl.h new file mode 100644 index 0000000000000000000000000000000000000000..5d61fc8a687186b36a86cef74ec23e4d6d5da5fb --- /dev/null +++ b/include/linux/fwctl.h @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES + */ +#ifndef __LINUX_FWCTL_H +#define __LINUX_FWCTL_H +#include +#include +#include +#include + +struct fwctl_device; +struct fwctl_uctx; + +/** + * struct fwctl_ops - Driver provided operations + * + * fwctl_unregister() will wait until all excuting ops are completed before it + * returns. Drivers should be mindful to not let their ops run for too long as + * it will block device hot unplug and module unloading. + */ +struct fwctl_ops { + /** + * @device_type: The drivers assigned device_type number. This is uABI. + */ + enum fwctl_device_type device_type; + /** + * @uctx_size: The size of the fwctl_uctx struct to allocate. The first + * bytes of this memory will be a fwctl_uctx. The driver can use the + * remaining bytes as its private memory. + */ + size_t uctx_size; + /** + * @open_uctx: Called when a file descriptor is opened before the uctx + * is ever used. + */ + int (*open_uctx)(struct fwctl_uctx *uctx); + /** + * @close_uctx: Called when the uctx is destroyed, usually when the FD + * is closed. + */ + void (*close_uctx)(struct fwctl_uctx *uctx); + /** + * @info: Implement FWCTL_INFO. Return a kmalloc() memory that is copied + * to out_device_data. On input length indicates the size of the user + * buffer on output it indicates the size of the memory. The driver can + * ignore length on input, the core code will handle everything. + */ + void *(*info)(struct fwctl_uctx *uctx, size_t *length); + /** + * @fw_rpc: Implement FWCTL_RPC. Deliver rpc_in/in_len to the FW and + * return the response and set out_len. rpc_in can be returned as the + * response pointer. Otherwise the returned pointer is freed with + * kvfree(). + */ + void *(*fw_rpc)(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope, + void *rpc_in, size_t in_len, size_t *out_len); +}; + +/** + * struct fwctl_device - Per-driver registration struct + * @dev: The sysfs (class/fwctl/fwctlXX) device + * + * Each driver instance will have one of these structs with the driver private + * data following immediately after. This struct is refcounted, it is freed by + * calling fwctl_put(). + */ +struct fwctl_device { + struct device dev; + /* private: */ + struct cdev cdev; + + /* Protect uctx_list */ + struct mutex uctx_list_lock; + struct list_head uctx_list; + /* + * Protect ops, held for write when ops becomes NULL during unregister, + * held for read whenever ops is loaded or an ops function is running. + */ + struct rw_semaphore registration_lock; + const struct fwctl_ops *ops; +}; + +struct fwctl_device *_fwctl_alloc_device(struct device *parent, + const struct fwctl_ops *ops, + size_t size); +/** + * fwctl_alloc_device - Allocate a fwctl + * @parent: Physical device that provides the FW interface + * @ops: Driver ops to register + * @drv_struct: 'struct driver_fwctl' that holds the struct fwctl_device + * @member: Name of the struct fwctl_device in @drv_struct + * + * This allocates and initializes the fwctl_device embedded in the drv_struct. + * Upon success the pointer must be freed via fwctl_put(). Returns a 'drv_struct + * \*' on success, NULL on error. + */ +#define fwctl_alloc_device(parent, ops, drv_struct, member) \ + ({ \ + static_assert(__same_type(struct fwctl_device, \ + ((drv_struct *)NULL)->member)); \ + static_assert(offsetof(drv_struct, member) == 0); \ + (drv_struct *)_fwctl_alloc_device(parent, ops, \ + sizeof(drv_struct)); \ + }) + +static inline struct fwctl_device *fwctl_get(struct fwctl_device *fwctl) +{ + get_device(&fwctl->dev); + return fwctl; +} +static inline void fwctl_put(struct fwctl_device *fwctl) +{ + put_device(&fwctl->dev); +} +DEFINE_FREE(fwctl, struct fwctl_device *, if (_T) fwctl_put(_T)); + +int fwctl_register(struct fwctl_device *fwctl); +void fwctl_unregister(struct fwctl_device *fwctl); + +/** + * struct fwctl_uctx - Per user FD context + * @fwctl: fwctl instance that owns the context + * + * Every FD opened by userspace will get a unique context allocation. Any driver + * private data will follow immediately after. + */ +struct fwctl_uctx { + struct fwctl_device *fwctl; + /* private: */ + /* Head at fwctl_device::uctx_list */ + struct list_head uctx_list_entry; +}; + +#endif diff --git a/include/linux/panic.h b/include/linux/panic.h index d0d592e55173b3a2d49f2434003df09f5d1a4dd9..adfe053cf6b24e6e022ef9dfe4ae12c8f319a1c0 100644 --- a/include/linux/panic.h +++ b/include/linux/panic.h @@ -75,7 +75,8 @@ static inline void set_arch_panic_timeout(int timeout, int arch_default_timeout) #define TAINT_AUX 16 #define TAINT_RANDSTRUCT 17 #define TAINT_TEST 18 -#define TAINT_FLAGS_COUNT 19 +#define TAINT_FWCTL 19 +#define TAINT_FLAGS_COUNT 20 #define TAINT_FLAGS_MAX ((1UL << TAINT_FLAGS_COUNT) - 1) struct taint_flag { diff --git a/include/linux/ummu_core.h b/include/linux/ummu_core.h index eda283f7b5246f05ca1ad762569fbc4e67712896..29d0952e35e7ac013cdf96159358db389db44fb5 100644 --- a/include/linux/ummu_core.h +++ b/include/linux/ummu_core.h @@ -25,18 +25,41 @@ #define UMMU_DEV_READ 2 #define UMMU_DEV_ATOMIC 4 +/** + * enum eid_type - the eid type + * + * @EID_NONE: nommal EID type + * @EID_BYPASS: ummu address translations are bypassed + * @EID_TYPE_MAX: max of eid type + */ enum eid_type { EID_NONE = 0, EID_BYPASS, EID_TYPE_MAX, }; +/** + * enum tid_alloc_mode - tid different allocated mode + * + * @TID_ALLOC_TRANSPARENT: use pasid as tid, no need to assign again + * @TID_ALLOC_ASSIGNED: pre-allocated tid, no need to assign again + * @TID_ALLOC_NORMAL: alloc tid normal + */ enum tid_alloc_mode { TID_ALLOC_TRANSPARENT = 0, TID_ALLOC_ASSIGNED = 1, TID_ALLOC_NORMAL = 2, }; +/** + * enum ummu_resource_type - SVA resource type + * + * @UMMU_BLOCK: mapt block + * @UMMU_QUEUE: permission queue + * @UMMU_QUEUE_LIST: permission queue for multi ummu + * @UMMU_CNT: ummu count + * @UMMU_TID_RES: tid resource + */ enum ummu_resource_type { UMMU_BLOCK, UMMU_QUEUE, @@ -51,6 +74,13 @@ enum default_tid_ops_types { TID_OPS_MAX, }; +/** + * enum ummu_register_type - ummu device register type + * + * @REGISTER_TYPE_GLOBAL: register as the global iommu device + * @REGISTER_TYPE_NORMAL: register to the iommu framework + * @REGISTER_TYPE_MAX: max of ummu device register type + */ enum ummu_register_type { REGISTER_TYPE_GLOBAL, REGISTER_TYPE_NORMAL, @@ -62,6 +92,12 @@ struct ummu_tid_manager; struct ummu_base_domain; struct ummu_core_device; +/** + * struct block_args - param related to mapt block + * @index: mapt block index + * @block_size_order: block size in PAGE_SIZE + * @out_addr: allocated physical address + */ struct block_args { u32 index; int block_size_order; @@ -75,6 +111,12 @@ struct block_args { KABI_RESERVE(6) }; +/** + * struct queue_args - param related to queue + * @pcmdq_base: base address of command queue + * @pcplq_base: base address of completion queue + * @ctrl_page: base address of permission queue + */ struct queue_args { phys_addr_t pcmdq_base; phys_addr_t pcplq_base; @@ -87,6 +129,13 @@ struct queue_args { KABI_RESERVE(5) }; +/** + * struct tid_args - param related to tid + * @pcmdq_order: base address of command queue + * @pcplq_order: base address of completion queue + * @blk_exp_size: block size in PAGE_SIZE + * @hw_cap: cap of hardware + */ struct tid_args { u8 pcmdq_order; u8 pcplq_order; @@ -100,6 +149,16 @@ struct tid_args { KABI_RESERVE(5) }; +/** + * struct resource_args - SVA resource related args + * @type: SVA resource type + * @block: arg related to mapt block + * @queue: arg related to mapt queue for UMMU_QUEUE + * @queues: arg related to mapt queue for UMMU_QUEUE_LIST in multi ummu mode + * @tid_res: tid resource + * @ummu_cnt: return value number of ummu + * @block_index: block index for release + */ struct resource_args { enum ummu_resource_type type; union { @@ -117,6 +176,10 @@ struct resource_args { KABI_RESERVE(3) }; +/** + * struct ummu_param - param related to tid + * @mode: mapt mode: table mode or entry mode + */ struct ummu_param { enum ummu_mapt_mode mode; @@ -129,6 +192,14 @@ struct ummu_param { KABI_RESERVE(7) }; +/** + * struct ummu_tid_param - param related to alloc tid + * @device: device pointer + * @mode: mapt mode: table mode or entry mode + * @alloc_mode: tid alloc mode + * @assign_tid: assigned tid, for TID_ALLOC_TRANSPARENT or TID_ALLOC_ASSIGNED + * @domain_type: more about domain-types in iommu.h + */ struct ummu_tid_param { struct device *device; enum ummu_mapt_mode mode; @@ -143,6 +214,13 @@ struct ummu_tid_param { KABI_RESERVE(5) }; +/** + * struct tdev_attr - attr for tdev + * @name: tdev name + * @dma_attr: dma mode + * @priv: private data pointer + * @priv_len: private data length + */ struct tdev_attr { const char *name; enum dev_dma_attr dma_attr; @@ -189,7 +267,7 @@ struct ummu_core_ops { }; /** - * ummu-core defined iommu device type + * struct ummu_core_device - ummu-core defined iommu device type * @list: used to link all ummu-core devices * @tid_manager: tid domain manager. * @iommu: iommu prototype @@ -212,6 +290,14 @@ struct ummu_core_device { KABI_RESERVE(8) }; +/** + * struct ummu_base_domain - domain info + * @domain: iommu domain + * @core_dev: ummu device + * @parent: point to father domain + * @list: base address of domain list + * @tid: token id + */ struct ummu_base_domain { struct iommu_domain domain; struct ummu_core_device *core_dev; @@ -224,6 +310,14 @@ struct ummu_base_domain { KABI_RESERVE(3) KABI_RESERVE(4) }; + +/** + * struct tid_ops - ummu ops for normal use, expand from iommu_ops + * @alloc_tid_manager: alloc manager for tid + * @free_tid_manager: free all tid and manager for tid + * @alloc_tid: alloc tid func + * @free_tid: free tid func + */ struct tid_ops { struct ummu_tid_manager *(*alloc_tid_manager)( struct ummu_core_device *core_device, u32 min_tid, @@ -239,6 +333,13 @@ struct tid_ops { KABI_RESERVE(4) }; +/** + * struct ummu_tid_manager - assigned tid manager + * @ops: ummu tid ops for normal use, expand from iommu_ops + * @token_ids: xarray of assigned tid + * @min_tid: min tid range for alloc + * @max_tid: max tid range for alloc + */ struct ummu_tid_manager { const struct tid_ops *ops; struct xarray token_ids; @@ -252,6 +353,12 @@ struct ummu_tid_manager { KABI_RESERVE(4) }; +/** + * struct ummu_core_tid_args - tid related args + * @tid_ops: ummu tid ops for normal use, expand from iommu_ops + * @max_tid: max tid range for alloc + * @min_tid: min tid range for alloc + */ struct ummu_core_tid_args { const struct tid_ops *tid_ops; u32 max_tid; @@ -265,6 +372,13 @@ struct ummu_core_tid_args { KABI_RESERVE(6) }; +/** + * struct ummu_core_init_args - ummu core init args + * @core_ops: the ummu device need ummu core ops capability + * @tid_args: parameters related to tid + * @iommu_ops: iommu_ops is mandatory + * @hwdev: related hwdev + */ struct ummu_core_init_args { const struct ummu_core_ops *core_ops; struct ummu_core_tid_args tid_args; @@ -276,7 +390,16 @@ struct ummu_core_init_args { KABI_RESERVE(3) }; -/* Memory traffic monitoring of the UB device */ +/** + * struct ummu_mpam - Memory traffic monitoring of the UB device + * @flags: flags, see constants above + * @eid: entity id + * @tid: tid + * @partid: mpam partition id + * @pmg: mpam pmg + * @s1mpam: 0 for ste mpam, 1 for cd mpam + * @user_mpam_en: 0 for ummu mpam, 1 for user mpam + */ struct ummu_mpam { #define UMMU_DEV_SET_MPAM (1 << 0) #define UMMU_DEV_GET_MPAM (1 << 1) @@ -326,7 +449,7 @@ static inline void tdev_attr_init(struct tdev_attr *attr) #ifdef CONFIG_UB_UMMU_CORE /* EID API */ /** - * Add a new EID to the UMMU. + * ummu_core_add_eid() - Add a new EID to the UMMU. * @guid: entity/device identity. * @eid: entity id to be added. * @type: eid type. @@ -335,7 +458,7 @@ static inline void tdev_attr_init(struct tdev_attr *attr) */ int ummu_core_add_eid(guid_t *guid, eid_t eid, enum eid_type type); /** - * Delete an EID from the UMMU. + * ummu_core_del_eid() - Delete an EID from the UMMU. * @guid: entity/device identity. * @eid: entity id to be deleted. * @type: eid type. @@ -344,7 +467,7 @@ void ummu_core_del_eid(guid_t *guid, eid_t eid, enum eid_type type); /* UMMU IOVA API */ /** - * Allocate a range of IOVA. The input iova size might be aligned. + * dma_alloc_iova() - Allocate a range of IOVA. The input iova size might be aligned. * @dev: related device. * @size: iova size. * @attrs: dma attributes. @@ -358,14 +481,14 @@ struct iova_slot *dma_alloc_iova(struct device *dev, size_t size, size_t *sizep); /** - * Free a range of IOVA. + * dma_free_iova() - Free a range of IOVA. * The API is not thread-safe. * @slot: iova slot, generated from dma_alloc_iova. */ void dma_free_iova(struct iova_slot *slot); /** - * Fill a range of IOVA. It allocates pages and maps pages to the iova. + * ummu_fill_pages() - Fill a range of IOVA. It allocates pages and maps pages to the iova. * The API is not thread-safe. * @slot: iova slot, generated from dma_alloc_iova. * @iova: iova start. @@ -376,7 +499,7 @@ void dma_free_iova(struct iova_slot *slot); int ummu_fill_pages(struct iova_slot *slot, dma_addr_t iova, unsigned long nr_pages); /** - * Drain a range of IOVA. It unmaps iova and releases pages. + * ummu_drain_pages() - Drain a range of IOVA. It unmaps iova and releases pages. * The API is not thread-safe. * @slot: iova slot, generated from dma_alloc_iova. * @iova: iova start. @@ -422,12 +545,15 @@ static inline int ummu_drain_pages(struct iova_slot *slot, dma_addr_t iova, #if IS_ENABLED(CONFIG_UB_UMMU_CORE_DRIVER) /* UMMU SVA API */ /** - * Grant va range permission to sva. + * ummu_sva_grant_range() - Grant va range permission to sva. * @sva: related sva handle. * @va: va start * @size: va size * @perm: permission * @cookie: struct ummu_token_info* + * + * .. code-block:: c + * * if (!cookie) { * do not use cookie check. * } else if (cookie->input == 0) { @@ -437,18 +563,20 @@ static inline int ummu_drain_pages(struct iova_slot *slot, dma_addr_t iova, * } else { * invalid para * } - * * Return: 0 on success, or an error. */ int ummu_sva_grant_range(struct iommu_sva *sva, void *va, size_t size, int perm, void *cookie); /** - * Ungrant va range permission from sva. + * ummu_sva_ungrant_range() - Ungrant va range permission from sva. * @sva: related sva handle. * @va: va start * @size: va size * @cookie: va related cookie,struct ummu_token_info* + * + * .. code-block:: c + * * if (!cookie) { * do not use cookie check. * } else { @@ -461,7 +589,7 @@ int ummu_sva_ungrant_range(struct iommu_sva *sva, void *va, size_t size, void *cookie); /** - * Get tid from dev or sva. + * ummu_get_tid() - Get tid from dev or sva. * @dev: related device. * @sva: if sva is set, return sva mode related tid; otherwise * return the dma mode tid. @@ -472,7 +600,7 @@ int ummu_sva_ungrant_range(struct iommu_sva *sva, void *va, size_t size, int ummu_get_tid(struct device *dev, struct iommu_sva *sva, u32 *tidp); /** - * Get iommu_domain by tid and dev. + * ummu_core_get_domain_by_tid() - Get iommu_domain by tid and dev. * @dev: related device. * @tid: tid * @@ -482,7 +610,7 @@ struct iommu_domain *ummu_core_get_domain_by_tid(struct device *dev, u32 tid); /** - * Check whether the UMMU works in ksva mode. + * ummu_is_ksva() - Check whether the UMMU works in ksva mode. * @domain: related iommu domain * * Return: true or false. @@ -490,7 +618,7 @@ struct iommu_domain *ummu_core_get_domain_by_tid(struct device *dev, bool ummu_is_ksva(struct iommu_domain *domain); /** - * Check whether the UMMU works in sva mode. + * ummu_is_sva() - Check whether the UMMU works in sva mode. * @domain: related iommu domain * * Return: true or false. @@ -498,10 +626,13 @@ bool ummu_is_ksva(struct iommu_domain *domain); bool ummu_is_sva(struct iommu_domain *domain); /** - * Bind device to a process mm. + * ummu_sva_bind_device() - Bind device to a process mm. * @dev: related device. * @mm: process memory management. * @drvdata: ummu_param related to tid. + * + * .. code-block:: c + * * if (!drvdata) { * sva is in the bypass mapt mode. * } else { @@ -514,7 +645,7 @@ struct iommu_sva *ummu_sva_bind_device(struct device *dev, struct mm_struct *mm, struct ummu_param *drvdata); /** - * Bind device to kernel mm. + * ummu_ksva_bind_device() - Bind device to kernel mm. * @dev: related device. * @drvdata: ummu_param related to tid. ksva doesn't support bypass mapt. * @@ -527,50 +658,55 @@ void ummu_ksva_unbind_device(struct iommu_sva *handle); /* UMMU CORE API */ /** - * Initialiase ummu core device. + * ummu_core_device_init() - Initialiase ummu core device. * @ummu_core: ummu core device. * @args: ummu core init args. + * * UMMU driver should carefully choose the args based on its requirement. * iommu_ops is mandatory. * a. the ummu device need tid allocation capability. + * * a.1 default tid strategies satisfy the ummu device * -> set tid_ops form ummu_core_tid_ops[TID_OPS_MAX] * a.2 default tid strategies do not satisfy the ummu device * -> implement a new tid_ops in the driver. + * * b. the ummu device need ummu core ops capability. * -> set core_ops. + * * c. the ummu device has related hwdev. * -> set hwdev. */ int ummu_core_device_init(struct ummu_core_device *ummu_core, struct ummu_core_init_args *args); /** - * Deinitialiase ummu core device. + * ummu_core_device_deinit() - Deinitialiase ummu core device. * @ummu_core: ummu core device. */ void ummu_core_device_deinit(struct ummu_core_device *ummu_core); /** - * Register ummu core device to the ummu framework. + * ummu_core_device_register() - Register ummu core device to the ummu framework. * @ummu_core: ummu core device. * @type: register type. - REGISTER_TYPE_GLOBAL: register the ummu device as the global device, - The ummu device will be the device handle all request. - e.g. 1. add_eid/del_eid 2. provide ubus iommu ops. etc. - - REGISTER_TYPE_NORMAL: follow the iommu_device register. will not be - related to the global device. it work as a normal iommu device. + * + * REGISTER_TYPE_GLOBAL: register the ummu device as the global device, + * The ummu device will be the device handle all request. + * e.g. 1. add_eid/del_eid 2. provide ubus iommu ops. etc. + * + * REGISTER_TYPE_NORMAL: follow the iommu_device register. will not be + * related to the global device. it work as a normal iommu device. */ int ummu_core_device_register(struct ummu_core_device *ummu_core, enum ummu_register_type type); /** - * Unregister ummu core device from the ummu framework. + * ummu_core_device_unregister() - Unregister ummu core device from the ummu framework. * @dev: the ummu_core device tid belongs to. */ void ummu_core_device_unregister(struct ummu_core_device *dev); /** - * Invalidate ummu global configuration by tid. + * ummu_core_invalidate_cfg_table() - Invalidate ummu global configuration by tid. * @tid: tid * Return: 0 on success, or an error. */ @@ -578,7 +714,7 @@ int ummu_core_invalidate_cfg_table(u32 tid); /* UMMU TID API */ /** - * Alloc a tid from ummu framework, and alloc related pasid. + * ummu_core_alloc_tid() - Alloc a tid from ummu framework, and alloc related pasid. * @dev: the allocated tid will be attached to. * @drvdata: ummu_tid_param related to tid * @tidp: the allocated tid returned here. @@ -589,14 +725,14 @@ int ummu_core_alloc_tid(struct ummu_core_device *dev, struct ummu_tid_param *drvdata, u32 *tidp); /** - * Free a tid to ummu framework. + * ummu_core_free_tid() - Free a tid to ummu framework. * @dev: the ummu_core device tid belongs to. * @tid: token id. */ void ummu_core_free_tid(struct ummu_core_device *dev, u32 tid); /** - * Get mapt_mode related to the tid. + * ummu_core_get_mapt_mode() - Get mapt_mode related to the tid. * @dev: the ummu_core device tid belongs to. * @tid: token id. * @@ -606,7 +742,7 @@ enum ummu_mapt_mode ummu_core_get_mapt_mode(struct ummu_core_device *dev, u32 tid); /** - * Get device related to the tid. + * ummu_core_get_device() - Get device related to the tid. * It will increase the ref count of the device. * @dev: the ummu_core device tid belongs to. * @tid: token id. @@ -617,7 +753,7 @@ struct device *ummu_core_get_device(struct ummu_core_device *dev, u32 tid); void ummu_core_put_device(struct device *dev); /** - * Allocate a virtual device to hold a tid. + * ummu_core_alloc_tdev() - Allocate a virtual device to hold a tid. * @attr: attributes of tdev * @ptid: tid pointer * Return: device on success or NULL error. @@ -625,7 +761,7 @@ void ummu_core_put_device(struct device *dev); struct device *ummu_core_alloc_tdev(struct tdev_attr *attr, u32 *ptid); /** - * Free the virtual device + * ummu_core_free_tdev() - Free the virtual device * @dev: Return value allocated by ummu_core_alloc_tdev * * Return: 0 on success or an error. @@ -633,7 +769,7 @@ struct device *ummu_core_alloc_tdev(struct tdev_attr *attr, u32 *ptid); int ummu_core_free_tdev(struct device *dev); /** - * Get ummu_tid_type related to the tid. + * ummu_core_get_tid_type() - Get ummu_tid_type related to the tid. * @dev: the ummu_core device tid belongs to. * @tid: token id. * @tid_type: out param, ummu_tid_type diff --git a/include/uapi/fwctl/fwctl.h b/include/uapi/fwctl/fwctl.h new file mode 100644 index 0000000000000000000000000000000000000000..bddd8d19695c48bb6f6d7643bc56e193bb0c7e5b --- /dev/null +++ b/include/uapi/fwctl/fwctl.h @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. + */ +#ifndef _UAPI_FWCTL_H +#define _UAPI_FWCTL_H + +#include +#include + +#define FWCTL_TYPE 0x9A + +/** + * DOC: General ioctl format + * + * The ioctl interface follows a general format to allow for extensibility. Each + * ioctl is passed a structure pointer as the argument providing the size of + * the structure in the first u32. The kernel checks that any structure space + * beyond what it understands is 0. This allows userspace to use the backward + * compatible portion while consistently using the newer, larger, structures. + * + * ioctls use a standard meaning for common errnos: + * + * - ENOTTY: The IOCTL number itself is not supported at all + * - E2BIG: The IOCTL number is supported, but the provided structure has + * non-zero in a part the kernel does not understand. + * - EOPNOTSUPP: The IOCTL number is supported, and the structure is + * understood, however a known field has a value the kernel does not + * understand or support. + * - EINVAL: Everything about the IOCTL was understood, but a field is not + * correct. + * - ENOMEM: Out of memory. + * - ENODEV: The underlying device has been hot-unplugged and the FD is + * orphaned. + * + * As well as additional errnos, within specific ioctls. + */ +enum { + FWCTL_CMD_BASE = 0, + FWCTL_CMD_INFO = 0, + FWCTL_CMD_RPC = 1, +}; + +enum fwctl_device_type { + FWCTL_DEVICE_TYPE_ERROR = 0, + FWCTL_DEVICE_TYPE_UB = 5, +}; + +/** + * struct fwctl_info - ioctl(FWCTL_INFO) + * @size: sizeof(struct fwctl_info) + * @flags: Must be 0 + * @out_device_type: Returns the type of the device from enum fwctl_device_type + * @device_data_len: On input the length of the out_device_data memory. On + * output the size of the kernel's device_data which may be larger or + * smaller than the input. Maybe 0 on input. + * @out_device_data: Pointer to a memory of device_data_len bytes. Kernel will + * fill the entire memory, zeroing as required. + * + * Returns basic information about this fwctl instance, particularly what driver + * is being used to define the device_data format. + */ +struct fwctl_info { + __u32 size; + __u32 flags; + __u32 out_device_type; + __u32 device_data_len; + __aligned_u64 out_device_data; +}; +#define FWCTL_INFO _IO(FWCTL_TYPE, FWCTL_CMD_INFO) + +/** + * enum fwctl_rpc_scope - Scope of access for the RPC + * + * Refer to fwctl.rst for a more detailed discussion of these scopes. + */ +enum fwctl_rpc_scope { + /** + * @FWCTL_RPC_CONFIGURATION: Device configuration access scope + * + * Read/write access to device configuration. When configuration + * is written to the device it remains in a fully supported state. + */ + FWCTL_RPC_CONFIGURATION = 0, + /** + * @FWCTL_RPC_DEBUG_READ_ONLY: Read only access to debug information + * + * Readable debug information. Debug information is compatible with + * kernel lockdown, and does not disclose any sensitive information. For + * instance exposing any encryption secrets from this information is + * forbidden. + */ + FWCTL_RPC_DEBUG_READ_ONLY = 1, + /** + * @FWCTL_RPC_DEBUG_WRITE: Writable access to lockdown compatible debug information + * + * Allows write access to data in the device which may leave a fully + * supported state. This is intended to permit intensive and possibly + * invasive debugging. This scope will taint the kernel. + */ + FWCTL_RPC_DEBUG_WRITE = 2, + /** + * @FWCTL_RPC_DEBUG_WRITE_FULL: Write access to all debug information + * + * Allows read/write access to everything. Requires CAP_SYS_RAW_IO, so + * it is not required to follow lockdown principals. If in doubt + * debugging should be placed in this scope. This scope will taint the + * kernel. + */ + FWCTL_RPC_DEBUG_WRITE_FULL = 3, +}; + +/** + * struct fwctl_rpc - ioctl(FWCTL_RPC) + * @size: sizeof(struct fwctl_rpc) + * @scope: One of enum fwctl_rpc_scope, required scope for the RPC + * @in_len: Length of the in memory + * @out_len: Length of the out memory + * @in: Request message in device specific format + * @out: Response message in device specific format + * + * Deliver a Remote Procedure Call to the device FW and return the response. The + * call's parameters and return are marshaled into linear buffers of memory. Any + * errno indicates that delivery of the RPC to the device failed. Return status + * originating in the device during a successful delivery must be encoded into + * out. + * + * The format of the buffers matches the out_device_type from FWCTL_INFO. + */ +struct fwctl_rpc { + __u32 size; + __u32 scope; + __u32 in_len; + __u32 out_len; + __aligned_u64 in; + __aligned_u64 out; +}; +#define FWCTL_RPC _IO(FWCTL_TYPE, FWCTL_CMD_RPC) + +#endif diff --git a/include/uapi/fwctl/ub_fwctl.h b/include/uapi/fwctl/ub_fwctl.h new file mode 100644 index 0000000000000000000000000000000000000000..38787e5cc8ca7d9d34fe1e4777b85fdbe8b84053 --- /dev/null +++ b/include/uapi/fwctl/ub_fwctl.h @@ -0,0 +1,314 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note*/ +/* + * Copyright(c) 2025 HiSilicon Technologies CO., Limited. All rights reserved. + */ + +#ifndef _UAPI_UB_FWCTL_H_ +#define _UAPI_UB_FWCTL_H_ + +#include + +/** + * struct fwctl_rpc_ub_in - ioctl(FWCTL_RPC) input + * @rpc_cmd: user specified opcode + * @data_size: Length of @data + * @version: Version passed in by the user + * @rsvd: reserved + * @data: user inputs specified input data + */ +struct fwctl_rpc_ub_in { + __u32 rpc_cmd; + __u32 data_size; + __u32 version; + __u32 rsvd; + __u32 data[] __counted_by(data_size); +}; + +/** + * struct fwctl_rpc_ub_out - ioctl(FWCTL_RPC) output + * @retval: The value returned when querying data with an error message + * @data_size: Length of @data + * @data: data transmitted to users + */ +struct fwctl_rpc_ub_out { + int retval; + __u32 data_size; + __u32 data[]; +}; + +/** + * enum ub_fwctl_cmdrpc_type - Type of access for the RPC + * + * Refer to fwctl.rst for a more detailed discussion of these scopes. + */ +enum ub_fwctl_cmdrpc_type { + /** + * @UTOOL_CMD_QUERY_NL: Query all registers at the NL layer + */ + UTOOL_CMD_QUERY_NL = 0x0001, + /** + * @UTOOL_CMD_QUERY_NL_PKT_STATS: Query NL layer PKT_STATE related registers + */ + UTOOL_CMD_QUERY_NL_PKT_STATS = 0x0002, + /** + * @UTOOL_CMD_QUERY_NL_SSU_STATS: Query NL layer SSU_STATS related registers + */ + UTOOL_CMD_QUERY_NL_SSU_STATS = 0x0003, + /** + * @UTOOL_CMD_QUERY_NL_ABN: Query NL layer NL_ABN related registers + */ + UTOOL_CMD_QUERY_NL_ABN = 0x0004, + + /** + * @UTOOL_CMD_QUERY_TP: Query all registers at the TP layer + */ + UTOOL_CMD_QUERY_TP = 0x0021, + /** + * @UTOOL_CMD_QUERY_TP_PKT_STATS: Query TP layer PKT_STATE related registers + */ + UTOOL_CMD_QUERY_TP_PKT_STATS = 0x0022, + /** + * @UTOOL_CMD_QUERY_TP_TX_ROUTE: Query TP layer TX_ROUTE related registers + */ + UTOOL_CMD_QUERY_TP_TX_ROUTE = 0x0023, + /** + * @UTOOL_CMD_QUERY_TP_ABN_STATS: Query TP layer ABN_STATS related registers + */ + UTOOL_CMD_QUERY_TP_ABN_STATS = 0x0024, + /** + * @UTOOL_CMD_QUERY_TP_RX_BANK: Query TP layer RX_BANK related registers + */ + UTOOL_CMD_QUERY_TP_RX_BANK = 0x0025, + + /** + * @UTOOL_CMD_QUERY_DL: Query all registers at the DL layer + */ + UTOOL_CMD_QUERY_DL = 0x0011, + /** + * @UTOOL_CMD_QUERY_DL_PKT_STATS: Query DL layer PKT_STATS related registers + */ + UTOOL_CMD_QUERY_DL_PKT_STATS = 0x0012, + /** + * @UTOOL_CMD_QUERY_DL_LINK_STATUS: Query DL layer LINK_STATUS related registers + */ + UTOOL_CMD_QUERY_DL_LINK_STATUS = 0x0013, + /** + * @UTOOL_CMD_QUERY_DL_LANE: Query DL layer LANE related registers + */ + UTOOL_CMD_QUERY_DL_LANE = 0x0014, + /** + * @UTOOL_CMD_QUERY_DL_BIT_ERR: Query DL layer BIT_ERR related registers + */ + UTOOL_CMD_QUERY_DL_BIT_ERR = 0x0015, + /** + * @UTOOL_CMD_QUERY_DL_LINK_TRACE: Query DL layer LINK_TRACE related registers + */ + UTOOL_CMD_QUERY_DL_LINK_TRACE = 0x0016, + /** + * @UTOOL_CMD_QUERY_DL_BIST: Query DL layer BIST related registers + */ + UTOOL_CMD_QUERY_DL_BIST = 0x0017, + /** + * @UTOOL_CMD_CONF_DL_BIST: Config DL layer BIST related registers + */ + UTOOL_CMD_CONF_DL_BIST = 0x0018, + /** + * @UTOOL_CMD_QUERY_DL_BIST_ERR: Query DL layer BIST_ERR related registers + */ + UTOOL_CMD_QUERY_DL_BIST_ERR = 0x0019, + + /** + * @UTOOL_CMD_QUERY_TA: Query all registers at the TA layer + */ + UTOOL_CMD_QUERY_TA = 0x0031, + /** + * @UTOOL_CMD_QUERY_TA_PKT_STATS: Query TA layer PKT_STATS related registers + */ + UTOOL_CMD_QUERY_TA_PKT_STATS = 0x0032, + /** + * @UTOOL_CMD_QUERY_TA_ABN_STATS: Query TA layer ABN_STATS related registers + */ + UTOOL_CMD_QUERY_TA_ABN_STATS = 0x0033, + + /** + * @UTOOL_CMD_QUERY_BA: Query all registers at the BA layer + */ + UTOOL_CMD_QUERY_BA = 0x0041, + /** + * @UTOOL_CMD_QUERY_BA_PKT_STATS: Query BA layer PKT_STATS related registers + */ + UTOOL_CMD_QUERY_BA_PKT_STATS = 0x0042, + /** + * @UTOOL_CMD_QUERY_BA_MAR: Query BA layer MAR related registers + */ + UTOOL_CMD_QUERY_BA_MAR = 0x0043, + /** + * @UTOOL_CMD_QUERY_BA_MAR_TABLE: Query BA layer MAR_TABLE related registers + */ + UTOOL_CMD_QUERY_BA_MAR_TABLE = 0x0044, + /** + * @UTOOL_CMD_QUERY_BA_MAR_CYC_EN: Query BA layer MAR_CYC_EN related registers + */ + UTOOL_CMD_QUERY_BA_MAR_CYC_EN = 0x0045, + /** + * @UTOOL_CMD_CONF_BA_MAR_CYC_EN: Config BA layer MAR_CYC_EN related registers + */ + UTOOL_CMD_CONF_BA_MAR_CYC_EN = 0x0046, + /** + * @UTOOL_CMD_CONFIG_BA_MAR_PEFR_STATS: Config BA layer MAR_PEFR_STATS related registers + */ + UTOOL_CMD_CONFIG_BA_MAR_PEFR_STATS = 0x0047, + /** + * @UTOOL_CMD_QUERY_BA_MAR_PEFR_STATS: Query BA layer MAR_PEFR_STATS related registers + */ + UTOOL_CMD_QUERY_BA_MAR_PEFR_STATS = 0x0048, + + /** + * @UTOOL_CMD_QUERY_QOS: Query QOS related registers + */ + UTOOL_CMD_QUERY_QOS = 0x0051, + + /** + * @UTOOL_CMD_QUERY_SCC_VERSION: Query the scc version + */ + UTOOL_CMD_QUERY_SCC_VERSION = 0x0061, + /** + * @UTOOL_CMD_QUERY_SCC_LOG: Query the scc log + */ + UTOOL_CMD_QUERY_SCC_LOG = 0x0062, + /** + * @UTOOL_CMD_QUERY_SCC_DEBUG_EN: Query the scc debug switch + */ + UTOOL_CMD_QUERY_SCC_DEBUG_EN = 0x0063, + /** + * @UTOOL_CMD_CONF_SCC_DEBUG_EN: Config the scc debug switch + */ + UTOOL_CMD_CONF_SCC_DEBUG_EN = 0x0064, + + /** + * @UTOOL_CMD_QUERY_MSGQ_QUE_STATS: Query MSGQ layer QUE_STATS related registers + */ + UTOOL_CMD_QUERY_MSGQ_QUE_STATS = 0x0071, + /** + * @UTOOL_CMD_QUERY_MSGQ_ENTRY: Query MSGQ layer ENTRY related registers + */ + UTOOL_CMD_QUERY_MSGQ_ENTRY = 0x0072, + + /** + * @UTOOL_CMD_QUERY_QUEUE: Query QUEUE information + */ + UTOOL_CMD_QUERY_QUEUE = 0x0073, + + /** + * @UTOOL_CMD_QUERY_PORT_INFO: Query information about the specified port + */ + UTOOL_CMD_QUERY_PORT_INFO = 0x0081, + /** + * @UTOOL_CMD_QUERY_IO_DIE_PORT_INFO: Query port-related information about the specified + * io die + */ + UTOOL_CMD_QUERY_IO_DIE_PORT_INFO = 0x0082, + + /** + * @UTOOL_CMD_QUERY_UBOMMU: Query UBOMMU related information + */ + UTOOL_CMD_QUERY_UBOMMU = 0x0091, + + /** + * @UTOOL_CMD_QUERY_UMMU_ALL: Query all information of UMMU + */ + UTOOL_CMD_QUERY_UMMU_ALL = 0x00A1, + /** + * @UTOOL_CMD_QUERY_UMMU_SYNC: Query information of UMMU SYNC + */ + UTOOL_CMD_QUERY_UMMU_SYNC = 0x00A2, + /** + * @UTOOL_CMD_CONFIG_UMMU_SYNC: Config information of UMMU SYNC + */ + UTOOL_CMD_CONFIG_UMMU_SYNC = 0x00A3, + + /** + * @UTOOL_CMD_QUERY_ECC_2B: Query information of ECC 2B + */ + UTOOL_CMD_QUERY_ECC_2B = 0x00B1, + + /** + * @UTOOL_CMD_QUERY_LOOPBACK: Query information of loopback + */ + UTOOL_CMD_QUERY_LOOPBACK = 0x00D1, + /** + * @UTOOL_CMD_CONF_LOOPBACK: Configure specified loopback mode + */ + UTOOL_CMD_CONF_LOOPBACK = 0x00D2, + /** + * @UTOOL_CMD_QUERY_PRBS_EN: Query PRBS switch status + */ + UTOOL_CMD_QUERY_PRBS_EN = 0x00D3, + /** + * @UTOOL_CMD_CONF_PRBS_EN: Config PRBS switch + */ + UTOOL_CMD_CONF_PRBS_EN = 0x00D4, + /** + * @UTOOL_CMD_QUERY_PRBS_RESULT: Query PRBS error count result + */ + UTOOL_CMD_QUERY_PRBS_RESULT = 0x00D5, + + /** + * @UTOOL_CMD_QUERY_DUMP: Dump all register data + */ + UTOOL_CMD_QUERY_DUMP = 0xFFFE, + + /** + * @UTOOL_CMD_QUERY_MAX: Maximum Command Code + */ + UTOOL_CMD_QUERY_MAX, +}; + +/** + * struct fwctl_pkt_in_enable - ioctl(FWCTL_RPC) input + * @enable: The value of param '-e' + */ +struct fwctl_pkt_in_enable { + __u8 enable; +}; + +/** + * struct fwctl_pkt_in_table - ioctl(FWCTL_RPC) input + * @port_id: The value of param '-p' + * @table_num: Length of the table + * @index: The value of param '-i' + */ +struct fwctl_pkt_in_table { + __u32 port_id; + __u32 table_num; + __u32 index; +}; + +/** + * struct fwctl_pkt_in_enable - ioctl(FWCTL_RPC) input + * @port_id: The value of param '-p' + */ +struct fwctl_pkt_in_port { + __u32 port_id; +}; + +/** + * struct fwctl_pkt_in_enable - ioctl(FWCTL_RPC) input + * @index: The value of param '-i' + */ +struct fwctl_pkt_in_index { + __u32 index; +}; + +/** + * struct fwctl_pkt_in_enable - ioctl(FWCTL_RPC) input + * @ummu_id: The value of param '-u' + * @value: The value of param '-e' + */ +struct fwctl_pkt_in_ummuid_value { + __u32 ummu_id; + __u32 value; +}; + +#endif + diff --git a/include/uapi/ub/cdma/cdma_abi.h b/include/uapi/ub/cdma/cdma_abi.h new file mode 100644 index 0000000000000000000000000000000000000000..681854ed9765b0a1c200d974a3d06264cf82ec78 --- /dev/null +++ b/include/uapi/ub/cdma/cdma_abi.h @@ -0,0 +1,420 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef _UAPI_UB_CDMA_CDMA_ABI_H_ +#define _UAPI_UB_CDMA_CDMA_ABI_H_ + +#include + +/* cdma ioctl cmd */ +#define CDMA_IOC_MAGIC 'C' +#define CDMA_SYNC _IOWR(CDMA_IOC_MAGIC, 0, struct cdma_ioctl_hdr) + +/* cdma event ioctl cmd */ +#define CDMA_EVENT_CMD_MAGIC 'F' +#define JFAE_CMD_GET_ASYNC_EVENT 0 +#define JFCE_CMD_WAIT_EVENT 0 + +#define CDMA_CMD_GET_ASYNC_EVENT \ + _IOWR(CDMA_EVENT_CMD_MAGIC, JFAE_CMD_GET_ASYNC_EVENT, struct cdma_cmd_async_event) +#define CDMA_CMD_WAIT_JFC \ + _IOWR(CDMA_EVENT_CMD_MAGIC, JFCE_CMD_WAIT_EVENT, struct cdma_cmd_jfce_wait_args) + +#define CDMA_ADDR_SHIFT 32 +#define CDMA_DOORBELL_OFFSET 0x80 + +#define CDMA_JFC_DB_CI_IDX_M GENMASK(21, 0) +#define CDMA_JFC_DB_VALID_OWNER_M 1 +#define CDMA_INTER_ERR 1 +#define CDMA_SRC_IDX_SHIFT 16 +#define CDMA_MAX_JFCE_EVENT_CNT 72 + +#define MAP_COMMAND_MASK 0xff +#define MAP_INDEX_MASK 0xffffff +#define MAP_INDEX_SHIFT 8 + +/* cdma queue cfg deault value */ +#define CDMA_TYPICAL_RNR_RETRY 7 +#define CDMA_TYPICAL_ERR_TIMEOUT 2 /* 0:128ms 1:1s 2:8s 3:64s */ + +#define CDMA_CQE_STATUS_NUM 7 +#define CDMA_CQE_SUB_STATUS_NUM 5 + +enum dma_cr_status { + DMA_CR_SUCCESS = 0, + DMA_CR_UNSUPPORTED_OPCODE_ERR, + DMA_CR_LOC_LEN_ERR, + DMA_CR_LOC_OPERATION_ERR, + DMA_CR_LOC_ACCESS_ERR, + DMA_CR_REM_RESP_LEN_ERR, + DMA_CR_REM_UNSUPPORTED_REQ_ERR, + DMA_CR_REM_OPERATION_ERR, + DMA_CR_REM_ACCESS_ABORT_ERR, + DMA_CR_ACK_TIMEOUT_ERR, + DMA_CR_RNR_RETRY_CNT_EXC_ERR, + DMA_CR_WR_FLUSH_ERR, + DMA_CR_WR_SUSPEND_DONE, + DMA_CR_WR_FLUSH_ERR_DONE, + DMA_CR_WR_UNHANDLED, + DMA_CR_LOC_DATA_POISON, + DMA_CR_REM_DATA_POISON, +}; + +enum db_mmap_type { + CDMA_MMAP_JFC_PAGE, + CDMA_MMAP_JETTY_DSQE +}; + +enum cdma_cmd { + CDMA_CMD_QUERY_DEV_INFO, + CDMA_CMD_CREATE_CTX, + CDMA_CMD_DELETE_CTX, + CDMA_CMD_CREATE_CTP, + CDMA_CMD_DELETE_CTP, + CDMA_CMD_CREATE_JFS, + CDMA_CMD_DELETE_JFS, + CDMA_CMD_REGISTER_SEG, + CDMA_CMD_UNREGISTER_SEG, + CDMA_CMD_CREATE_QUEUE, + CDMA_CMD_DELETE_QUEUE, + CDMA_CMD_CREATE_JFC, + CDMA_CMD_DELETE_JFC, + CDMA_CMD_CREATE_JFCE, + CDMA_CMD_MAX +}; + +enum { + CQE_FOR_SEND, + CQE_FOR_RECEIVE +}; + +enum hw_cqe_opcode { + HW_CQE_OPC_SEND = 0x00, + HW_CQE_OPC_SEND_WITH_IMM = 0x01, + HW_CQE_OPC_SEND_WITH_INV = 0x02, + HW_CQE_OPC_WRITE_WITH_IMM = 0x03, + HW_CQE_OPC_ERR = 0xff +}; + +struct cdma_ioctl_hdr { + __u32 command; + __u32 args_len; + __u64 args_addr; +}; + +struct cdma_create_jfs_ucmd { + __u64 buf_addr; + __u32 buf_len; + __u64 db_addr; + __u64 idx_addr; + __u32 idx_len; + __u64 jetty_addr; + __u32 sqe_bb_cnt; + __u32 jetty_type; + __u32 non_pin; + __u32 jfs_id; + __u32 queue_id; + __u32 tid; +}; + +struct cdma_cmd_udrv_priv { + __u64 in_addr; + __u32 in_len; + __u64 out_addr; + __u32 out_len; +}; + +struct cdma_cmd_create_jfs_args { + struct { + __u32 depth; + __u32 flag; + __u32 eid_idx; + __u8 priority; + __u8 max_sge; + __u8 max_rsge; + __u8 retry_cnt; + __u8 rnr_retry; + __u8 err_timeout; + __u32 jfc_id; + __u32 queue_id; + __u32 rmt_eid; + __u32 pld_token_id; + __u32 tpn; + __u64 dma_jfs; /* dma jfs pointer */ + __u32 trans_mode; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } in; + struct { + __u32 id; + __u32 depth; + __u8 max_sge; + __u8 max_rsge; + __u64 handle; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } out; + struct cdma_cmd_udrv_priv udata; +}; + +struct cdma_cmd_async_event { + __u64 event_data; + __u32 event_type; +}; + +struct cdma_cmd_delete_jfs_args { + struct { + __u32 jfs_id; + __u64 handle; + __u32 queue_id; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } in; + struct { + __u32 rsv_bitmap; + __u32 rsvd[4]; + } out; +}; + +struct cdma_cmd_create_ctp_args { + struct { + __u32 scna; + __u32 dcna; + __u32 eid_idx; + __u32 upi; + __u64 dma_tp; + __u32 seid; + __u32 deid; + __u32 queue_id; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } in; + struct { + __u32 tpn; + __u64 handle; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } out; +}; + +struct cdma_cmd_delete_ctp_args { + struct { + __u32 tpn; + __u64 handle; + __u32 queue_id; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } in; + struct { + __u32 rsv_bitmap; + __u32 rsvd[4]; + } out; +}; + +struct cdma_cmd_create_jfce_args { + struct { + __u32 rsv_bitmap; + __u32 rsvd[4]; + } in; + struct { + int fd; + int id; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } out; +}; + +struct cdma_cmd_create_jfc_args { + struct { + __u32 depth; /* in terms of CQEBB */ + int jfce_fd; + int jfce_id; + __u32 ceqn; + __u32 queue_id; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } in; + struct { + __u32 id; + __u32 depth; + __u64 handle; /* handle of the allocated jfc obj in kernel */ + __u32 rsv_bitmap; + __u32 rsvd[4]; + } out; + struct cdma_cmd_udrv_priv udata; +}; + +struct cdma_cmd_delete_jfc_args { + struct { + __u32 jfcn; + __u64 handle; /* handle of jfc */ + __u32 queue_id; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } in; + struct { + __u32 comp_events_reported; + __u32 async_events_reported; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } out; +}; + +struct cdma_cmd_register_seg_args { + struct { + __u64 addr; + __u64 len; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } in; + struct { + __u64 handle; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } out; +}; + +struct cdma_cmd_unregister_seg_args { + struct { + __u64 handle; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } in; + struct { + __u32 rsv_bitmap; + __u32 rsvd[4]; + } out; +}; + +struct dev_eid { + __u32 dw0; + __u32 dw1; + __u32 dw2; + __u32 dw3; +}; + +struct eu_info { + __u32 eid_idx; + struct dev_eid eid; + __u32 upi; +}; + +struct cdma_device_cap { + __u32 max_jfc; + __u32 max_jfs; + __u32 max_jfc_depth; + __u32 max_jfs_depth; + __u32 max_jfs_inline_len; + __u32 max_jfs_sge; + __u32 max_jfs_rsge; + __u64 max_msg_size; + __u32 max_atomic_size; + __u16 trans_mode; + __u32 ceq_cnt; + __u32 max_eid_cnt; + __u64 page_size_cap; +}; + +struct cdma_device_attr { +#define CDMA_MAX_EU_NUM 64 + __u8 eu_num; + struct dev_eid eid; + struct eu_info eu; + struct eu_info eus[CDMA_MAX_EU_NUM]; + struct cdma_device_cap dev_cap; +}; + +struct cdma_cmd_query_device_attr_args { + struct { + __u32 rsv_bitmap; + __u32 rsvd[4]; + } in; + struct { + struct cdma_device_attr attr; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } out; +}; + +struct cdma_create_context_args { + struct { + __u32 rsv_bitmap; + __u32 rsvd[4]; + } in; + struct { + __u8 cqe_size; + __u8 dwqe_enable; + int async_fd; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } out; +}; + +struct cdma_jfc_db { + __u32 ci : 24; + __u32 notify : 1; + __u32 arm_sn : 2; + __u32 type : 1; + __u32 rsv1 : 4; + __u32 jfcn : 20; + __u32 rsv2 : 12; +}; + +struct cdma_create_jfc_ucmd { + __u64 buf_addr; + __u32 buf_len; + __u64 db_addr; + __u32 mode; + __u32 tid; +}; + +struct cdma_cmd_create_queue_args { + struct { + __u32 queue_depth; + __u32 dcna; + __u32 rmt_eid; + __u8 priority; + __u64 user_ctx; + __u32 trans_mode; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } in; + struct { + int queue_id; + __u64 handle; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } out; +}; + +struct cdma_cmd_delete_queue_args { + struct { + __u32 queue_id; + __u64 handle; + __u32 rsv_bitmap; + __u32 rsvd[4]; + } in; + struct { + __u32 rsv_bitmap; + __u32 rsvd[4]; + } out; +}; + +struct cdma_cmd_jfce_wait_args { + struct { + __u32 max_event_cnt; + int time_out; + } in; + struct { + __u32 event_cnt; + __u64 event_data[CDMA_MAX_JFCE_EVENT_CNT]; + } out; +}; + +enum jfc_poll_state { + JFC_OK, + JFC_EMPTY, + JFC_POLL_ERR, +}; + +#endif diff --git a/include/uapi/ub/urma/udma/udma_abi.h b/include/uapi/ub/urma/udma/udma_abi.h new file mode 100644 index 0000000000000000000000000000000000000000..5859f5254b5e4543cc34cbe5d313abb8a6bb7601 --- /dev/null +++ b/include/uapi/ub/urma/udma/udma_abi.h @@ -0,0 +1,184 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef _UAPI_UB_UMDK_URMA_UDMA_UDMA_ABI_H_ +#define _UAPI_UB_UMDK_URMA_UDMA_UDMA_ABI_H_ + +#include + +#define MAP_COMMAND_MASK 0xff +#define MAP_INDEX_MASK 0xffffff +#define MAP_INDEX_SHIFT 8 + +#define UDMA_SEGMENT_ACCESS_GUARD (1UL << 5) + +#define UDMA_CQE_COALESCE_SHIFT 10 +#define UDMA_CQE_COALESCE_CNT_MAX (1 << UDMA_CQE_COALESCE_SHIFT) + +#define UDMA_CQE_PERIOD_0 0 +#define UDMA_CQE_PERIOD_4 4 +#define UDMA_CQE_PERIOD_16 16 +#define UDMA_CQE_PERIOD_64 64 +#define UDMA_CQE_PERIOD_256 256 +#define UDMA_CQE_PERIOD_1024 1024 +#define UDMA_CQE_PERIOD_4096 4096 +#define UDMA_CQE_PERIOD_16384 16384 + +#define UDMA_JFC_HW_DB_OFFSET 0x40 + +#define UDMA_DOORBELL_OFFSET 0x80 + +#define UDMA_JETTY_DSQE_OFFSET 0x1000 + +#define UDMA_DB_SIZE 64U + +#define UDMA_SRC_IDX_SHIFT 16 +#define UDMA_IMM_DATA_SHIFT 32 +#define UDMA_JFC_DB_VALID_OWNER_M 1 +#define UDMA_ADDR_SHIFT 32 + +#define UDMA_INTER_ERR 1 +#define UDMA_CQE_DEFAULT_SUBSTATUS 0 + +#define UDMA_MAX_GRANT_SIZE 0xFFFFFFFFF000 + +#define UDMA_TID_SHIFT 8U +#define UDMA_MAX_TID 0xFFFFFU + +enum udma_jetty_type { + UDMA_CACHE_LOCK_DWQE_JETTY_TYPE, + UDMA_CCU_JETTY_TYPE, + UDMA_NORMAL_JETTY_TYPE, + UDMA_URMA_NORMAL_JETTY_TYPE, + UDMA_JETTY_TYPE_MAX +}; + +enum cr_direct { + CR_SEND, + CR_RECV, +}; + +enum cr_jetty { + CR_IS_NOT_JETTY, + CR_IS_JETTY, +}; + +struct udma_create_jetty_ucmd { + __aligned_u64 buf_addr; + __u32 buf_len; + __u32 jfr_id; + __aligned_u64 db_addr; + __aligned_u64 idx_addr; + __u32 idx_len; + __u32 sqe_bb_cnt; + __aligned_u64 jetty_addr; + __u32 pi_type : 1; + __u32 non_pin : 1; + __u32 is_hugepage : 1; + __u32 rsv : 29; + __u32 jetty_type; + __aligned_u64 jfr_sleep_buf; + __u32 jfs_id; + __u32 rsv1; +}; + +struct udma_create_jfc_ucmd { + __aligned_u64 buf_addr; + __u32 buf_len; + __u32 mode; /* 0: normal, 1: user stars, 2: kernel stars */ + __aligned_u64 db_addr; + __u32 is_hugepage : 1; + __u32 rsv : 31; + __u32 rsv1; +}; + +struct udma_create_ctx_resp { + __u32 cqe_size : 8; + __u32 dwqe_enable : 1; + __u32 reduce_enable : 1; + __u32 dump_aux_info : 1; + __u32 hugepage_enable : 1; + __u32 rsv : 20; + __u32 ue_id; + __u32 chip_id; + __u32 die_id; + __u32 jfr_sge; + __u32 rsv1; +}; + +struct udma_create_jfr_resp { + __u32 jfr_caps; + __u32 rsv; +}; + +enum db_mmap_type { + UDMA_MMAP_JFC_PAGE, + UDMA_MMAP_JETTY_DSQE, + UDMA_MMAP_HUGEPAGE, +}; + +enum { + UDMA_CQ_DB, + UDMA_CQ_ARM_DB, +}; + +struct udma_jfc_db { + __u32 ci : 24; + __u32 notify : 1; + __u32 arm_sn : 2; + __u32 type : 1; + __u32 rsv1 : 4; + __u32 jfcn : 20; + __u32 rsv2 : 12; +}; + +enum udma_db_type { + UDMA_JFR_TYPE_DB, + UDMA_JFC_TYPE_DB, + UDMA_JFR_PAYLOAD, + UDMA_DB_TYPE_NUM, +}; + +enum jfc_poll_state { + JFC_OK, + JFC_EMPTY, + JFC_POLL_ERR, +}; + +enum { + CQE_FOR_SEND, + CQE_FOR_RECEIVE, +}; + +enum { + UDMA_CQE_SUCCESS = 0x00, + UDMA_CQE_UNSUPPORTED_OPCODE = 0x01, + UDMA_CQE_LOCAL_OP_ERR = 0x02, + UDMA_CQE_REMOTE_OP_ERR = 0x03, + UDMA_CQE_TRANSACTION_RETRY_COUNTER_ERR = 0x04, + UDMA_CQE_TRANSACTION_ACK_TIMEOUT_ERR = 0x05, + UDMA_JETTY_WORK_REQUEST_FLUSH = 0x06, +}; + +enum { + UDMA_CQE_LOCAL_LENGTH_ERR = 0x01, + UDMA_CQE_LOCAL_ACCESS_ERR = 0x02, + UDMA_CQE_REM_RSP_LENGTH_ERR = 0x03, + UDMA_CQE_LOCAL_DATA_POISON = 0x04, +}; + +enum { + UDMA_CQE_REM_UNSUPPORTED_REQ_ERR = 0x01, + UDMA_CQE_REM_ACCESS_ERR = 0x02, + UDMA_CQE_REM_DATA_POISON = 0x04, +}; + +enum hw_cqe_opcode { + HW_CQE_OPC_SEND = 0x00, + HW_CQE_OPC_SEND_WITH_IMM = 0x01, + HW_CQE_OPC_SEND_WITH_INV = 0x02, + HW_CQE_OPC_WRITE_WITH_IMM = 0x03, + HW_CQE_OPC_ERR = 0xff, +}; + +#endif /* _UAPI_UB_UMDK_URMA_UDMA_UDMA_ABI_H_ */ diff --git a/include/ub/cdma/cdma_api.h b/include/ub/cdma/cdma_api.h new file mode 100644 index 0000000000000000000000000000000000000000..51acd722a74d2d3b3ba27a5799585e82a8b6577b --- /dev/null +++ b/include/ub/cdma/cdma_api.h @@ -0,0 +1,239 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2025 HiSilicon Technologies Co., Ltd. All rights reserved. */ + +#ifndef _UB_CDMA_CDMA_API_H_ +#define _UB_CDMA_CDMA_API_H_ + +#include +#include + +/** + * struct dma_device - DMA device structure + * @attr: CDMA device attribute info: EID, UPI etc + * @ref_cnt: reference count for adding a context to device + * @private_data: cdma context resoucres pointer + * @rsv_bitmap: reserved field bitmap + * @rsvd: reserved field array + */ +struct dma_device { + struct cdma_device_attr attr; + atomic_t ref_cnt; + void *private_data; + u32 rsv_bitmap; + u32 rsvd[4]; +}; + +enum dma_cr_opcode { + DMA_CR_OPC_SEND = 0x00, + DMA_CR_OPC_SEND_WITH_IMM, + DMA_CR_OPC_SEND_WITH_INV, + DMA_CR_OPC_WRITE_WITH_IMM, +}; + +/** + * union dma_cr_flag - DMA completion record flag + * @bs: flag bit value structure + * @value: flag value + */ +union dma_cr_flag { + struct { + u8 s_r : 1; /* indicate CR stands for sending or receiving */ + u8 jetty : 1; /* indicate id in the CR stands for jetty or JFS */ + u8 suspend_done : 1; /* suspend done flag */ + u8 flush_err_done : 1; /* flush error done flag */ + u8 reserved : 4; + } bs; + u8 value; +}; + +/** + * struct dma_cr - DMA completion record structure + * @status: completion record status + * @user_ctx: user private data information, optional + * @opcode: DMA operation code + * @flag: completion record flag + * @completion_len: the number of bytes transferred + * @local_id: local JFS ID + * @remote_id: remote JFS ID, not in use for now + * @tpn: transport number + * @rsv_bitmap: reserved field bitmap + * @rsvd: reserved field array + */ +struct dma_cr { + enum dma_cr_status status; + u64 user_ctx; + enum dma_cr_opcode opcode; + union dma_cr_flag flag; + u32 completion_len; + u32 local_id; + u32 remote_id; + u32 tpn; + u32 rsv_bitmap; + u32 rsvd[4]; +}; + +/** + * struct queue_cfg - DMA queue config structure + * @queue_depth: queue depth + * @priority: the priority of JFS, ranging from [0, 15] + * @user_ctx: user private data information, optional + * @dcna: remote device CNA + * @rmt_eid: remote device EID + * @rsv_bitmap: reserved field bitmap + * @rsvd: reserved field array + */ +struct queue_cfg { + u32 queue_depth; + u8 priority; + u64 user_ctx; + u32 dcna; + struct dev_eid rmt_eid; + u32 trans_mode; + u32 rsv_bitmap; + u32 rsvd[6]; +}; + +/** + * struct dma_seg - DMA segment structure + * @handle: segment recouse handle + * @sva: payload virtual address + * @len: payload data length + * @tid: payload token id + * @token_value: not used for now + * @token_value_valid: not used for now + * @rsv_bitmap: reserved field bitmap + * @rsvd: reserved field array + */ +struct dma_seg { + u64 handle; + u64 sva; + u64 len; + u32 tid; /* data valid only in bit 0-19 */ + u32 token_value; + bool token_value_valid; + u32 rsv_bitmap; + u32 rsvd[4]; +}; + +struct dma_seg_cfg { + u64 sva; + u64 len; + u32 token_value; + bool token_value_valid; + u32 rsv_bitmap; + u32 rsvd[4]; +}; + +/** + * struct dma_context - DMA context structure + * @dma_dev: DMA device pointer + * @tid: token id for segment + */ +struct dma_context { + struct dma_device *dma_dev; + u32 tid; /* data valid only in bit 0-19 */ +}; + +enum dma_status { + DMA_STATUS_OK, + DMA_STATUS_INVAL, +}; + +/** + * struct dma_cas_data - DMA CAS data structure + * @compare_data: compare data, length <= 8B: CMP value, length > 8B: data address + * @swap_data: swap data, length <= 8B: swap value, length > 8B: data address + * @rsv_bitmap: reserved field bitmap + * @rsvd: reserved field array + */ +struct dma_cas_data { + u64 compare_data; + u64 swap_data; + u32 rsv_bitmap; + u32 rsvd[4]; +}; + +/** + * struct dma_notify_data - DMA write witch notify data structure + * @notify_seg: notify segment pointer + * @notify_data: notify data value + * @rsv_bitmap: reserved field bitmap + * @rsvd: reserved field array + */ +struct dma_notify_data { + struct dma_seg *notify_seg; + u64 notify_data; + u32 rsv_bitmap; + u32 rsvd[4]; +}; + +/** + * struct dma_client - DMA register client structure + * @list_node: client list + * @client_name: client name pointer + * @add: add DMA resource function pointer + * @remove: remove DMA resource function pointer + * @stop: stop DMA operation function pointer + * @rsv_bitmap: reserved field bitmap + * @rsvd: reserved field array + */ +struct dma_client { + struct list_head list_node; + char *client_name; + int (*add)(u32 eid); + void (*remove)(u32 eid); + void (*stop)(u32 eid); + u32 rsv_bitmap; + u32 rsvd[4]; +}; + +struct dma_device *dma_get_device_list(u32 *num_devices); + +void dma_free_device_list(struct dma_device *dev_list, u32 num_devices); + +struct dma_device *dma_get_device_by_eid(struct dev_eid *eid); + +int dma_create_context(struct dma_device *dma_dev); + +void dma_delete_context(struct dma_device *dma_dev, int handle); + +int dma_alloc_queue(struct dma_device *dma_dev, int ctx_id, + struct queue_cfg *cfg); + +void dma_free_queue(struct dma_device *dma_dev, int queue_id); + +struct dma_seg *dma_register_seg(struct dma_device *dma_dev, int ctx_id, + struct dma_seg_cfg *cfg); + +void dma_unregister_seg(struct dma_device *dma_dev, struct dma_seg *dma_seg); + +struct dma_seg *dma_import_seg(struct dma_seg_cfg *cfg); + +void dma_unimport_seg(struct dma_seg *dma_seg); + +enum dma_status dma_write(struct dma_device *dma_dev, struct dma_seg *rmt_seg, + struct dma_seg *local_seg, int queue_id); + +enum dma_status dma_write_with_notify(struct dma_device *dma_dev, + struct dma_seg *rmt_seg, + struct dma_seg *local_seg, int queue_id, + struct dma_notify_data *data); + +enum dma_status dma_read(struct dma_device *dma_dev, struct dma_seg *rmt_seg, + struct dma_seg *local_seg, int queue_id); + +enum dma_status dma_cas(struct dma_device *dma_dev, struct dma_seg *rmt_seg, + struct dma_seg *local_seg, int queue_id, + struct dma_cas_data *data); + +enum dma_status dma_faa(struct dma_device *dma_dev, struct dma_seg *rmt_seg, + struct dma_seg *local_seg, int queue_id, u64 add); + +int dma_poll_queue(struct dma_device *dma_dev, int queue_id, u32 cr_cnt, + struct dma_cr *cr); + +int dma_register_client(struct dma_client *client); + +void dma_unregister_client(struct dma_client *client); + +#endif diff --git a/include/ub/ubase/ubase_comm_cmd.h b/include/ub/ubase/ubase_comm_cmd.h index 7e78860d77781861080f022720075d4a5dafece2..cadc707a23e6ed23428bb1b48aa6c59583d11fbc 100644 --- a/include/ub/ubase/ubase_comm_cmd.h +++ b/include/ub/ubase/ubase_comm_cmd.h @@ -30,13 +30,21 @@ enum ubase_opcode_type { /* Generic commands */ UBASE_OPC_QUERY_FW_VER = 0x0001, UBASE_OPC_QUERY_CTL_INFO = 0x0003, + UBASE_OPC_NOTIFY_DRV_CAPS = 0x0007, UBASE_OPC_QUERY_COMM_RSRC_PARAM = 0x0030, UBASE_OPC_QUERY_NIC_RSRC_PARAM = 0x0031, UBASE_OPC_QUERY_LINK_STATUS = 0x0032, UBASE_OPC_CFG_MTU = 0x0033, UBASE_OPC_QUERY_NET_GUID = 0x0035, UBASE_OPC_STATS_MAC_ALL = 0x0038, + UBASE_OPC_DFX_REG_NUM = 0x0039, + UBASE_OPC_DFX_DL_REG = 0x0040, + UBASE_OPC_DFX_NL_REG = 0x0042, + UBASE_OPC_DFX_BA_REG = 0x0043, + UBASE_OPC_DFX_TP_REG = 0x0044, + UBASE_OPC_DFX_TA_REG = 0x0045, UBASE_OPC_QUERY_BUS_EID = 0x0047, + UBASE_OPC_QUERY_UBCL_CONFIG = 0x0050, /* NL commands */ UBASE_OPC_CFG_VL_MAP = 0x2206, @@ -79,6 +87,7 @@ enum ubase_opcode_type { UBASE_OPC_QUERY_PORT_INFO = 0x6200, UBASE_OPC_QUERY_CHIP_INFO = 0x6201, UBASE_OPC_QUERY_FEC_STATS = 0x6202, + UBASE_OPC_HIMAC_RESET = 0x6302, /* Mailbox commands */ UBASE_OPC_POST_MB = 0x7000, @@ -94,12 +103,30 @@ enum ubase_opcode_type { UBASE_OPC_NOTIFY_UE_RESET = 0xF006, UBASE_OPC_QUERY_UE_RST_RDY = 0xF007, UBASE_OPC_RESET_DONE = 0xF008, + UBASE_OPC_VPORT_CTX = 0xF009, UBASE_OPC_DESTROY_CTX_RESOURCE = 0xF00D, UBASE_OPC_UE2UE_UBASE = 0xF00E, UBASE_OPC_ACTIVATE_REQ = 0xF00F, UBASE_OPC_ACTIVATE_RESP = 0xF010, }; +/** + * union ubase_mbox - ubase mailbox structure + * @in_param_l: input data storage address lower 32 bits + * @in_param_h: input data storage address high 32 bits + * @cmd: mailbox command + * @tag: queue id + * @seq_num: sequence number + * @event_en: 0-poll mode, 1-event mode + * @mbx_ue_id: mailbox ub entity id + * @rsv: reserved bits + * @status: mailbox command execution completion status, 0-success, 1-fail + * @hw_run: hardware running status, 0-not running, 1-running + * @rsv1: reserved bits + * @query_status:execution result of the mailbox query command, 0-success, 1-fail + * @query_hw_run: hardware running status of the mailbox query command, 0-not running, 1-running + * @query_rsv: reserved bits + */ union ubase_mbox { struct { /* MB 0 */ @@ -127,6 +154,13 @@ union ubase_mbox { }; }; +/** + * struct ubase_cmd_buf - ubase cmd buffer structure + * @opcode: cmdq opcode + * @is_read: read or write, true for read, false for write + * @data_size: valid length of data + * @data: data buffer + */ struct ubase_cmd_buf { u16 opcode; bool is_read; @@ -134,6 +168,13 @@ struct ubase_cmd_buf { void *data; }; +/** + * struct ubase_crq_event_nb - ubase crq event notification block structure + * @opcode: cmdq crq opcode + * @back: arbitrary registered pointer + * @crq_handler: cmdq crq handle function. dev: the struct member variable 'back', + * data: the crq message data, len: the crq message data length. + */ struct ubase_crq_event_nb { u16 opcode; void *back; diff --git a/include/ub/ubase/ubase_comm_ctrlq.h b/include/ub/ubase/ubase_comm_ctrlq.h index 7e772dcb074650b27d6fde540313012b50e00678..c50bfd60047f092e81240403cd293cd372e80639 100644 --- a/include/ub/ubase/ubase_comm_ctrlq.h +++ b/include/ub/ubase/ubase_comm_ctrlq.h @@ -28,16 +28,17 @@ enum ubase_ctrlq_ser_ver { }; enum ubase_ctrlq_ser_type { - UBASE_CTRLQ_SER_TYPE_TP_ACL = 0x01, - UBASE_CTRLQ_SER_TYPE_DEV_REGISTER = 0x02, - UBASE_CTRLQ_SER_TYPE_IP_ACL = 0x03, - UBASE_CTRLQ_SER_TYPE_QOS = 0x04, + UBASE_CTRLQ_SER_TYPE_TP_ACL = 0x01, + UBASE_CTRLQ_SER_TYPE_DEV_REGISTER = 0x02, + UBASE_CTRLQ_SER_TYPE_IP_ACL = 0x03, + UBASE_CTRLQ_SER_TYPE_QOS = 0x04, }; -enum ubase_ctrlq_opc_type { +enum ubase_ctrlq_opc_type_tp { UBASE_CTRLQ_OPC_CREATE_TP = 0x11, UBASE_CTRLQ_OPC_DESTROY_TP = 0x12, UBASE_CTRLQ_OPC_TP_FLUSH_DONE = 0x14, + UBASE_CTRLQ_OPC_CHECK_TP_ACTIVE = 0x15, }; enum ubase_ctrlq_opc_type_qos { @@ -45,23 +46,42 @@ enum ubase_ctrlq_opc_type_qos { UBASE_CTRLQ_OPC_QUERY_SL = 0x02, }; -enum ubase_ctrlq_opc_ip { - UBASE_CTRLQ_OPC_NOTIFY_IP = 0x01, - UBASE_CTRLQ_OPC_QUERY_IP = 0x02, +enum ubase_ctrlq_opc_type_ip { + UBASE_CTRLQ_OPC_NOTIFY_IP = 0x01, + UBASE_CTRLQ_OPC_QUERY_IP = 0x02, }; enum ubase_ctrlq_opc_type_dev_register { - UBASE_CTRLQ_OPC_CTRLQ_CTRL = 0x14, - UBASE_CTRLQ_OPC_UE_RESET_CTRL = 0x15, + UBASE_CTRLQ_OPC_UPDATE_SEID = 0x02, + UBASE_CTRLQ_OPC_UPDATE_UE_SEID_GUID = 0x03, + UBASE_CTRLQ_OPC_NOTIFY_RES_RATIO = 0x13, + UBASE_CTRLQ_OPC_CTRLQ_CTRL = 0x14, + UBASE_CTRLQ_OPC_UE_RESET_CTRL = 0x15, }; +/** + * struct ubase_ctrlq_msg - ubase ctrlq msg structure + * @service_ver: ctrlq service version + * @service_type: ctrlq service type + * @opcode: ctrlq opcode + * @need_resp: whether the message need a response + * @is_resp: whether the message is a response + * @resv: reserved bits + * @resp_ret: the return value of response message + * @resp_seq: response message sequence + * @in_size: input data buffer size + * @out_size: output data buffer size + * @in: input data buffer + * @out: output data buffer + */ struct ubase_ctrlq_msg { enum ubase_ctrlq_ser_ver service_ver; enum ubase_ctrlq_ser_type service_type; u8 opcode; u8 need_resp : 1; u8 is_resp : 1; - u8 resv : 6; + u8 is_async : 1; + u8 resv : 5; u8 resp_ret; /* must set when the is_resp field is true. */ u16 resp_seq; /* must set when the is_resp field is true. */ u16 in_size; @@ -70,6 +90,13 @@ struct ubase_ctrlq_msg { void *out; }; +/** + * struct ubase_ctrlq_event_nb - ubase ctrlq event notification block structure + * @service_type: ctrlq service type + * @opcode: ctrlq crq opcode + * @back: arbitrary registered pointer + * @crq_handler: ctrlq crq handle function + */ struct ubase_ctrlq_event_nb { u8 service_type; u8 opcode; @@ -80,7 +107,6 @@ struct ubase_ctrlq_event_nb { int ubase_ctrlq_send_msg(struct auxiliary_device *aux_dev, struct ubase_ctrlq_msg *msg); - int ubase_ctrlq_register_crq_event(struct auxiliary_device *aux_dev, struct ubase_ctrlq_event_nb *nb); void ubase_ctrlq_unregister_crq_event(struct auxiliary_device *aux_dev, diff --git a/include/ub/ubase/ubase_comm_debugfs.h b/include/ub/ubase/ubase_comm_debugfs.h index dc0bd30aa93bf2aa9b8bdfaf30da1d185115dd6e..4d17c98bdfba1dafb3c3ecf1404f730f3eb39e74 100644 --- a/include/ub/ubase/ubase_comm_debugfs.h +++ b/include/ub/ubase/ubase_comm_debugfs.h @@ -11,6 +11,14 @@ struct ubase_dbgfs; +/** + * struct ubase_dbg_dentry_info - ubase debugfs dentry information + * @name: dentry name + * @dentry: the dentry for this file + * @property: property supported by this dentry + * @support: function to determine whether to create this dentry. dev: struct device, + * property: the struct number variable 'property'. + */ struct ubase_dbg_dentry_info { const char *name; struct dentry *dentry; @@ -18,6 +26,18 @@ struct ubase_dbg_dentry_info { bool (*support)(struct device *dev, u32 property); }; +/** + * struct ubase_dbg_cmd_info - ubase debugfs cmd information + * @name: file name + * @dentry_index: dentry index + * @property: property supported by this file. + * @support: function to determine whether to create this file. dev: struct device, + * property: the struct number variable 'property. + * @init: init debugfs cmd file function. dev: struct device, dirs: struct + * ubase_dbg_dentry_info, dbgfs: struct ubase_dbgfs, idx: idxth file. + * @read_func: reading debugfs detailed implementation function. s: struct seq_file, + * data: data buffer + */ struct ubase_dbg_cmd_info { const char *name; int dentry_index; @@ -28,8 +48,14 @@ struct ubase_dbg_cmd_info { int (*read_func)(struct seq_file *s, void *data); }; +/** + * struct ubase_dbgfs - ubase debugfs data structure + * @dentry: debugfs root path + * @cmd_info: ubase debugfs cmd information + * @cmd_info_size: the size of cmd_info + */ struct ubase_dbgfs { - struct dentry *dentry; /* dbgfs root path */ + struct dentry *dentry; struct ubase_dbg_cmd_info *cmd_info; int cmd_info_size; }; diff --git a/include/ub/ubase/ubase_comm_dev.h b/include/ub/ubase/ubase_comm_dev.h index 81b2c98af17f4e3933bc8d6dd4995699433116bf..8dfbb2dc91bdb273609978db473e1a7af8176375 100644 --- a/include/ub/ubase/ubase_comm_dev.h +++ b/include/ub/ubase/ubase_comm_dev.h @@ -24,6 +24,8 @@ struct iova_slot; #error "UBASE_MAX_VL_NUM can't less than IEEE_8021QAZ_MAX_TCS" #endif +#define UBASE_NIC_MAX_VL_NUM (2) + #define UBASE_SUP_UBL BIT(0) #define UBASE_SUP_ETH BIT(1) #define UBASE_SUP_UNIC BIT(2) @@ -51,6 +53,46 @@ enum ubase_reset_stage { UBASE_RESET_STAGE_UP, }; +/** + * struct ubase_caps - ubase capabilities + * @num_ceq_vectors: completion event vectors number + * @num_aeq_vectors: asynchronous event vectors umber + * @num_misc_vectors: misc event vectors number + * @aeqe_depth: the depth of asynchronous event vector queue + * @ceqe_depth: the depth of completion event vector queue + * @aeqe_size: the size of asynchronous event vector queue element + * @ceqe_size: the size of completion event vector queue element + * @total_ue_num: ue number + * @public_jetty_cnt: public jetty count + * @vl_num: vl number + * @rsvd_jetty_cnt: reserved jetty count + * @req_vl: requested vl + * @resp_vl: response vl + * @packet_pattern_mode: packet pattern mode + * @ack_queue_num: ack queue number + * @oor_en: out of order receive, 0: disable 1: enable + * @reorder_queue_en: reorder queue enable, 0: disable 1: enable + * @on_flight_size: on flight packets size + * @reorder_cap: reorder capability + * @reorder_queue_shift: reorder queue shift + * @at_times: ack timeout + * @ue_num: the total number of ue and mue + * @mac_stats_num: mac stats number + * @logic_port_bitmap: logic port bitmap + * @ub_port_logic_id: ub port logic id + * @io_port_logic_id: io port logic id + * @io_port_id: io port id + * @nl_port_id: nl port id + * @chip_id: chip id + * @die_id: die id + * @ue_id: ub entity id + * @nl_id: nl id + * @tid: ub entity tid + * @eid: ub entity eid + * @upi: ub entity upi + * @ctl_no: ub controller id + * @fw_version: firmware version + */ struct ubase_caps { u16 num_ceq_vectors; u16 num_aeq_vectors; @@ -98,18 +140,47 @@ struct ubase_caps { u32 fw_version; }; +/** + * struct ubase_res_caps - ubase resource capbilities + * @max_cnt: the resource max count + * @start_idx: start index + * @reserved_cnt: reserved count + * @depth: the queue depth of the resource + */ struct ubase_res_caps { u32 max_cnt; u32 start_idx; - u32 reserved_cnt; u32 depth; }; +/** + * struct ubase_pmem_caps - ubase physical memory capabilities + * @dma_len: iova address sapce length + * @dma_addr: iova address + */ struct ubase_pmem_caps { u64 dma_len; dma_addr_t dma_addr; }; +/** + * struct ubase_adev_caps - ubase auxiliary device capabilities + * @jfs: jfs resource capabilities + * @jfr: jfr resource capabilities + * @jfc: jfc resource capabilities + * @tp: tp resource capabilities + * @tpg: tp group resource capabilities + * @pmem: physical memory capabilities + * @utp_port_bitmap: utp port bitmap + * @jtg_max_cnt: jetty group max count + * @rc_max_cnt: rc max count + * @rc_que_depth: rc queue depth + * @ccc_max_cnt: ccc max count + * @dest_addr_max_cnt: dest addr max count + * @seid_upi_max_cnt:seid upi max count + * @tpm_max_cnt: tpm max count + * @cqe_size: cqe size + */ struct ubase_adev_caps { struct ubase_res_caps jfs; struct ubase_res_caps jfr; @@ -121,13 +192,19 @@ struct ubase_adev_caps { u32 jtg_max_cnt; u32 rc_max_cnt; u32 rc_que_depth; - u32 ccc_max_cnt; - u32 dest_addr_max_cnt; - u32 seid_upi_max_cnt; - u32 tpm_max_cnt; u16 cqe_size; }; +/** + * struct ubase_ctx_buf_cap - ubase context buffer capabilities + * @dma_ctx_buf_ba: context buffer iova address + * @slot: iova slot + * @entry_size: context entry size + * @entry_cnt: context entry count + * @cnt_per_page_shift: context entry count per page shift + * @ctx_xa: context array + * @ctx_mutex: context mutex + */ struct ubase_ctx_buf_cap { dma_addr_t dma_ctx_buf_ba; /* pass to hw */ struct iova_slot *slot; @@ -138,6 +215,16 @@ struct ubase_ctx_buf_cap { struct mutex ctx_mutex; }; +/** + * struct ubase_ctx_buf - ubase context buffer information + * @jfs: jfs context buffer capability + * @jfr: jfr context buffer capability + * @jfc: jfc context buffer capability + * @jtg: jetty group context buffer capability + * @rc: rc context buffer capability + * @tp: tp context buffer capability + * @tpg: tp group context buffer capability + */ struct ubase_ctx_buf { struct ubase_ctx_buf_cap jfs; struct ubase_ctx_buf_cap jfr; @@ -150,16 +237,54 @@ struct ubase_ctx_buf { }; struct net_device; + +/** + * struct ubase_adev_com - ubase auxiliary device common information + * @adev: auxiliary device + * @netdev: network device + */ struct ubase_adev_com { struct auxiliary_device *adev; struct net_device *netdev; }; +/** + * struct ubase_resource_space - ubase resource space + * @addr_unmapped: unmapped address + * @addr: mapped address + */ struct ubase_resource_space { resource_size_t addr_unmapped; void __iomem *addr; }; +/** + * struct ubase_adev_qos - ubase auxiliary device qos information + * @rdma_vl_num: rdma vl number + * @rdma_tp_vl_num: rdma tp vl number + * @rdma_ctp_vl_num: rdma ctp vl number + * @rdma_tp_resp_vl_offset: rdma tp response vl offset, + * rdma_tp_resp_vl = rdma_ctp_resp_vl + rdma_tp_resp_vl_offset + * @rdma_ctp_resp_vl_offset: rdma ctp response vl offset, + * rdma_ctp_resp_vl = rdma_ctp_resp_vl + rdma_ctp_resp_vl_offset + * @max_vl: max vl number + * @resv: reserved bits + * @rdma_sl_num: rdma sl number + * @rdma_tp_sl_num: rdma tp sl number + * @rdma_ctp_sl_num: rdma ctp sl number + * @nic_sl_num: nic sl number + * @nic_vl_num: nic vl number + * @rdma_vl: rdma vl + * @rdma_tp_req_vl: rdma tp request vl + * @rdma_ctp_req_vl: rdma ctp request vl + * @rdma_sl: rdma sl + * @rdma_tp_sl: rdma tp sl + * @rdma_ctp_sl: rdma ctp sl + * @nic_sl: nic sl + * @nic_vl: nic vl + * @sl_vl: sl to vl mapping + * @rdma_dscp_vl: rdma dscp to vl mapping + */ struct ubase_adev_qos { /* udma/cdma resource */ u8 sl_num; @@ -192,6 +317,11 @@ struct ubase_adev_qos { u8 ue_sl_vl[UBASE_MAX_SL_NUM]; }; +/** + * struct ubase_ue_node - ubase ub entity list node + * @list: list head + * @bus_ue_id: bus ub entity id + */ struct ubase_ue_node { struct list_head list; u16 bus_ue_id; @@ -214,6 +344,11 @@ struct ubase_ue_caps { }; #define UBASE_BUS_EID_LEN 4 + +/** + * struct ubase_bus_eid - bus eid + * @eid: bus eid + */ struct ubase_bus_eid { u32 eid[UBASE_BUS_EID_LEN]; }; @@ -222,7 +357,6 @@ bool ubase_adev_ubl_supported(struct auxiliary_device *adev); bool ubase_adev_ctrlq_supported(struct auxiliary_device *adev); bool ubase_adev_eth_mac_supported(struct auxiliary_device *adev); bool ubase_adev_prealloc_supported(struct auxiliary_device *aux_dev); - struct ubase_resource_space *ubase_get_io_base(struct auxiliary_device *adev); struct ubase_resource_space *ubase_get_mem_base(struct auxiliary_device *adev); struct ubase_caps *ubase_get_dev_caps(struct auxiliary_device *adev); @@ -230,34 +364,27 @@ struct ubase_adev_caps *ubase_get_unic_caps(struct auxiliary_device *adev); struct ubase_adev_caps *ubase_get_udma_caps(struct auxiliary_device *adev); struct ubase_adev_caps *ubase_get_cdma_caps(struct auxiliary_device *adev); struct ubase_adev_qos *ubase_get_adev_qos(struct auxiliary_device *adev); - void ubase_reset_event(struct auxiliary_device *adev, enum ubase_reset_type reset_type); enum ubase_reset_stage ubase_get_reset_stage(struct auxiliary_device *adev); - void ubase_virt_register(struct auxiliary_device *adev, void (*virt_handler)(struct auxiliary_device *adev, u16 bus_ue_id, bool is_en)); void ubase_virt_unregister(struct auxiliary_device *adev); - void ubase_port_register(struct auxiliary_device *adev, void (*port_handler)(struct auxiliary_device *adev, bool link_up)); void ubase_port_unregister(struct auxiliary_device *adev); - void ubase_reset_register(struct auxiliary_device *adev, void (*reset_handler)(struct auxiliary_device *adev, enum ubase_reset_stage stage)); void ubase_reset_unregister(struct auxiliary_device *adev); - void ubase_activate_register(struct auxiliary_device *adev, void (*activate_handler)(struct auxiliary_device *adev, bool activate)); void ubase_activate_unregister(struct auxiliary_device *adev); - int ubase_activate_dev(struct auxiliary_device *adev); int ubase_deactivate_dev(struct auxiliary_device *adev); - int ubase_get_bus_eid(struct auxiliary_device *adev, struct ubase_bus_eid *eid); #endif diff --git a/include/ub/ubase/ubase_comm_eq.h b/include/ub/ubase/ubase_comm_eq.h index fe97de1dd1ac3baf58bd9b77335500ea688751e5..d1efad0a79b3a4f43b116ce2ac9e5c5aadf799f8 100644 --- a/include/ub/ubase/ubase_comm_eq.h +++ b/include/ub/ubase/ubase_comm_eq.h @@ -50,6 +50,13 @@ enum ubase_event_type { UBASE_EVENT_TYPE_MAX }; +/** + * struct ubase_event_nb - ubase event notification block + * @drv_type: auxiliary device driver type + * @event_type: event type + * @nb: notifier block + * @back: arbitrary registered pointer + */ struct ubase_event_nb { enum ubase_drv_type drv_type; u8 event_type; @@ -57,6 +64,20 @@ struct ubase_event_nb { void *back; }; +/** + * struct ubase_aeqe - asynchronous event interrupt queue elements + * @event_type: event type + * @sub_type: sub event type + * @rsv0: reserved bits + * @owner: owner bit + * @num: jfsn/jettyn/jfrn/jfcn/jtgn/tpn + * @rsv1: reserved bits + * @out_param: mailbox output parameter + * @seq_num: mailbox sequence number + * @status: mailbox status + * @event: aeqe event information + * @rsv: reserved bits + */ struct ubase_aeqe { u32 event_type : 8; u32 sub_type : 8; @@ -82,6 +103,12 @@ struct ubase_aeqe { u32 rsv[12]; }; +/** + * struct ubase_aeq_notify_info - aeq notification information + * @event_type: event type + * @sub_type: sub event type + * @aeqe: aeq elements + */ struct ubase_aeq_notify_info { u8 event_type; u8 sub_type; @@ -92,7 +119,6 @@ int ubase_event_register(struct auxiliary_device *adev, struct ubase_event_nb *cb); void ubase_event_unregister(struct auxiliary_device *adev, struct ubase_event_nb *cb); - int ubase_comp_register(struct auxiliary_device *adev, int (*comp_handler)(struct notifier_block *nb, unsigned long jfcn, void *data)); diff --git a/include/ub/ubase/ubase_comm_hw.h b/include/ub/ubase/ubase_comm_hw.h index ba3717fb16b377f9373e80be6df8d8550b4d5da8..2efac24e326856564ac7c2f8d8d3e3aa039f2406 100644 --- a/include/ub/ubase/ubase_comm_hw.h +++ b/include/ub/ubase/ubase_comm_hw.h @@ -19,6 +19,16 @@ #define UBASE_JTG_CTX_SIZE 8 #define UBASE_DESC_DATA_LEN 6 + +/** + * struct ubase_cmdq_desc - Command queue descriptor + * @opcode: Command opcode + * @flag: Command flag + * @bd_num: bd number. One bd is 32 bytes, and the first db is 24 bytes. + * @ret: Command return value + * @rsv: reserved + * @data: Command data + */ struct ubase_cmdq_desc { __le16 opcode; u8 flag; @@ -28,6 +38,16 @@ struct ubase_cmdq_desc { __le32 data[UBASE_DESC_DATA_LEN]; }; +/** + * struct ubase_cmdq_ring - Command ring queue information + * @ci: consumer indicator + * @pi: producer indicator + * @desc_num: descriptors number + * @tx_timeout: transmit timeout interval + * @desc_dma_addr: dma address of descriptors + * @desc: Command queue descriptor + * @lock: spinlock + */ struct ubase_cmdq_ring { u32 ci; u32 pi; @@ -38,11 +58,24 @@ struct ubase_cmdq_ring { spinlock_t lock; }; +/** + * struct ubase_cmdq - cmmand queue + * @csq: command send queue + * @crq: command receive queue + */ struct ubase_cmdq { struct ubase_cmdq_ring csq; struct ubase_cmdq_ring crq; }; +/** + * struct ubase_hw - hardware information + * @rs0_base: resource0 space base addr + * @io_base: io space base addr + * @mem_base: memory space base addr + * @cmdq: command queue + * @state: state of the hardware + */ struct ubase_hw { struct ubase_resource_space rs0_base; struct ubase_resource_space io_base; @@ -51,6 +84,13 @@ struct ubase_hw { unsigned long state; }; +/** + * struct ubase_mbx_event_context - mailbox event context + * @done: completion object to wait for event + * @result: mailbox execution result + * @out_param: mailbox output parameter + * @seq_num: mailbox sequence number + */ struct ubase_mbx_event_context { struct completion done; int result; diff --git a/include/ub/ubase/ubase_comm_mbx.h b/include/ub/ubase/ubase_comm_mbx.h index 26c625f80a778906eeedaa0e6c1f9b2f10b35ef3..9e7bbf1faf9ee47cca97852567c4bcf190e1b756 100644 --- a/include/ub/ubase/ubase_comm_mbx.h +++ b/include/ub/ubase/ubase_comm_mbx.h @@ -10,11 +10,23 @@ #include #include +/** + * struct ubase_cmd_mailbox - mailbox cmmand address + * @buf: virtual address + * @dma: dma address + */ struct ubase_cmd_mailbox { void *buf; dma_addr_t dma; }; +/** + * struct ubase_mbx_attr - mailbox attribute + * @tag: queue id + * @rsv: reserved bits + * @op: mailbox opcode + * @mbx_ue_id: mailbox ub entity id + */ struct ubase_mbx_attr { __le32 tag : 24; __le32 rsv : 8; @@ -78,11 +90,21 @@ enum ubase_mbox_opcode { struct ubase_cmd_mailbox *ubase_alloc_cmd_mailbox(struct auxiliary_device *aux_dev); void ubase_free_cmd_mailbox(struct auxiliary_device *aux_dev, struct ubase_cmd_mailbox *mailbox); - int ubase_hw_upgrade_ctx_ex(struct auxiliary_device *aux_dev, struct ubase_mbx_attr *attr, struct ubase_cmd_mailbox *mailbox); +/** + * ubase_fill_mbx_attr() - fill mailbox attribute + * @attr: mailbox attribute + * @tag: queue id + * @op: mailbox opcode + * @mbx_ue_id: mailbox ub entity id + * + * The function is used to assign 'tag', 'op', and 'mbx_ue_id' to 'struct ubase_mbx_attr'. + * + * Context: Process context. + */ static inline void ubase_fill_mbx_attr(struct ubase_mbx_attr *attr, u32 tag, u8 op, u8 mbx_ue_id) { diff --git a/include/ub/ubase/ubase_comm_qos.h b/include/ub/ubase/ubase_comm_qos.h index 01fc6942ca01d9d49c62b7ddde99eb3b56a75c56..daacd537658b3cea2b98c2a9752c3216a0dd9894 100644 --- a/include/ub/ubase/ubase_comm_qos.h +++ b/include/ub/ubase/ubase_comm_qos.h @@ -16,6 +16,13 @@ enum ubase_sl_sched_mode { UBASE_SL_DWRR = IEEE_8021QAZ_TSA_ETS, }; +/** + * struct ubase_sl_priqos - priority qos + * @port_bitmap: port bitmap + * @sl_bitmap: sl bitmap + * @weight: bandwidth weight + * @sch_mode: schedule mode + */ struct ubase_sl_priqos { u32 port_bitmap; u32 sl_bitmap; diff --git a/include/ub/ubase/ubase_comm_stats.h b/include/ub/ubase/ubase_comm_stats.h index fdafc18e6700ebb29104b264ef08cb6fb6c6c5f1..52a766e7bab0b754cd6b730e84f52d9235cfb2ea 100644 --- a/include/ub/ubase/ubase_comm_stats.h +++ b/include/ub/ubase/ubase_comm_stats.h @@ -210,6 +210,17 @@ struct ubase_eth_mac_stats { u64 rx_merge_frame_smd_error_pkts; }; +/** + * struct ubase_perf_stats_result - traffic bandwidth statistics results + * @valid: data valid flag, 0-invalid, 1-valid + * @resv0: reserved bits + * @port_id: port id + * @resv1: reserved bits + * @tx_port_bw: tx port bandwidth + * @rx_port_bw: rx port bandwidth + * @tx_vl_bw: tx vl bandwidth + * @rx_vl_bw: rx vl bandwidth + */ struct ubase_perf_stats_result { u8 valid : 1; u8 resv0 : 7; diff --git a/include/ub/ubus/ubus.h b/include/ub/ubus/ubus.h index a81d652a18ff6f84cf092f2a3d8cb9db9d22a627..ca3ba63c226aed02c5571a3a27b54265f104298c 100644 --- a/include/ub/ubus/ubus.h +++ b/include/ub/ubus/ubus.h @@ -193,7 +193,7 @@ struct ub_entity { u32 token_id; u32 token_value; - /* mue & ue info */ + /* MUE & UE info */ u8 is_mue; u16 total_ues; u16 num_ues; @@ -205,7 +205,7 @@ struct ub_entity { /* entity topology info */ struct list_head node; struct ub_bus_controller *ubc; - struct ub_entity *pue; /* ue/mue connected to their mue */ + struct ub_entity *pue; /* UE/MUE connected to their MUE */ int topo_rank; /* The levels of Breadth-First Search */ /* entity port info */ @@ -334,10 +334,10 @@ struct ub_dynids { * @shutdown: Hook into reboot_notifier_list (kernel/sys.c). * Intended to stop any idling operations. * @virt_configure: Optional driver callback to allow configuration of - * ues. This function is called to enable or disable ues. + * UEs. This function is called to enable or disable UEs. * @virt_notify: Optional driver callback to notify the driver about - * changes in ue status. This function is called - * when the status of a ue changes. + * changes in UE status. This function is called + * when the status of a UE changes. * @activate: Activate a specific entity. This function is called to * activate an entity by its index. * @deactivate: Deactivate a specific entity. This function is called to @@ -510,14 +510,14 @@ void ub_bus_type_iommu_ops_set(const struct iommu_ops *ops); const struct iommu_ops *ub_bus_type_iommu_ops_get(void); /** - * ub_get_ent_by_eid() - Searching for UB Devices by EID. + * ub_get_ent_by_eid() - Searching for UB entity by EID. * @eid: entity EID. * * Traverse the UB bus device linked list and search for the device with * the target EID. You need to call ub_entity_put() after using it. * * Context: Any context. - * Return: The device found, or NULL if not found. + * Return: The entity found, or NULL if not found. */ struct ub_entity *ub_get_ent_by_eid(unsigned int eid); @@ -566,8 +566,7 @@ struct ub_entity *ub_get_entity(unsigned int vendor, unsigned int entity, * @uent: UB entity. * @enable: Enable or disable. * - * Enables or disables the entity access bus and the path through which - * the bus accesses the entity. + * Enable or disable the communication channel between entity and user host. * * Context: Any context. */ @@ -588,31 +587,31 @@ int ub_set_user_info(struct ub_entity *uent); * ub_unset_user_info() - Deinitialize host information for the entity. * @uent: UB entity. * - * Clearing the Host Information of a entity. + * Clearing the host information of an entity. * * Context: Any context. */ void ub_unset_user_info(struct ub_entity *uent); /** - * ub_enable_entities() - Enable ues of mue in batches. - * @pue: UB mue. + * ub_enable_entities() - Enable UEs of MUE in batches. + * @pue: UB MUE. * @nums: Number of enabled entities. * - * Create ues in batches, initialize them, and add them to the system. + * Create and initialize UEs in batches and add to the system. * * Context: Any context. - * Return: 0 if success, or %-EINVAL if @pue type is not mue or nums over - * mue's total ue nums, or %-ENOMEM if the system is out of memory, + * Return: 0 if success, or %-EINVAL if @pue type is not MUE or nums over + * MUE's total UE nums, or %-ENOMEM if the system is out of memory, * or other failed negative values. */ int ub_enable_entities(struct ub_entity *pue, int nums); /** - * ub_disable_entities() - Disable ues of mue in batches. - * @pue: UB mue. + * ub_disable_entities() - Disable UEs of MUE in batches. + * @pue: UB MUE. * - * Remove all enabled ues under the mue from the system. + * Remove all enabled UEs under the MUE from the system. * * Context: Any context. */ @@ -620,29 +619,29 @@ void ub_disable_entities(struct ub_entity *pue); /** * ub_enable_ue() - Enable a single ue. - * @pue: UB mue. + * @pue: UB MUE. * @entity_idx: Number of the entity to be enabled. * - * Create a specified ue under mue, initialize the ue, + * Create a specified UE under MUE, initialize the ue, * and add it to the system. * * Context: Any context. - * Return: 0 if success, or %-EINVAL if @pue type is not mue or @entity_idx - * is no longer in the ue range of mue, or %-EEXIST if entity has been + * Return: 0 if success, or %-EINVAL if @pue type is not MUE or @entity_idx + * is no longer in the UE range of MUE, or %-EEXIST if entity has been * enabled, or other failed negative values. */ int ub_enable_ue(struct ub_entity *pue, int entity_idx); /** * ub_disable_ue() - Disable a single ue. - * @pue: UB mue. + * @pue: UB MUE. * @entity_idx: Number of the entity to be disabled. * - * Remove a specified ue. + * Remove a specified UE. * * Context: Any context. - * Return: 0 if success, or %-EINVAL if @pue type is not mue or @entity_idx - * is no longer in the ue range of mue, or %-ENODEV if entity hasn't + * Return: 0 if success, or %-EINVAL if @pue type is not MUE or @entity_idx + * is no longer in the UE range of MUE, or %-ENODEV if entity hasn't * been enabled. */ int ub_disable_ue(struct ub_entity *pue, int entity_idx); @@ -662,7 +661,7 @@ bool ub_get_entity_flex_en(void); * @uent: UB entity. * * Return the EID of bus instance if the entity has already been bound, - * or controller's EID. + * otherwise return controller's EID. * * Context: Any context. * Return: positive number if success, or %-EINVAL if @dev is %NULL, @@ -743,7 +742,7 @@ void ub_unregister_share_port(struct ub_entity *uent, u16 port_id, * ub_reset_entity() - Function entity level reset. * @ent: UB entity. * - * Reset a single entity without affecting other entities, If you want to reuse + * Reset a single entity without affecting other entities. If you want to reuse * the entity after reset, you need to re-initialize it. * * Context: Any context @@ -757,7 +756,7 @@ int ub_reset_entity(struct ub_entity *ent); * ub_device_reset() - Device level reset. * @ent: UB entity. * - * Reset Device, include all entities under the device, If you want to reuse + * Reset device, including all entities under the device. If you want to reuse * the device after reset, you need to re-initialize it. * * Context: Any context @@ -771,10 +770,10 @@ int ub_device_reset(struct ub_entity *ent); * @uent: UB entity. * @vdm_pld: Vendor private message payload context. * - * Send a vendor private message to the entity. Response will put in - * vdm_pld->rsp_pld, and will fill in vdm->rsp_pld_len. + * Send a vendor private message to the entity. Response will be put in + * vdm_pld->rsp_pld, and response length is stored in vdm->rsp_pld_len. * - * Context: Any context, It will take spin_lock_irqsave()/spin_unlock_restore() + * Context: Any context, it will take spin_lock_irqsave()/spin_unlock_restore() * Return: 0 if success, or %-EINVAL if parameters invalid, * or %-ENOMEM if system out of memory, or other failed negative values. */ @@ -796,10 +795,10 @@ unsigned int ub_irq_calc_affinity_vectors(unsigned int minvec, void ub_disable_intr(struct ub_entity *uent); /** - * ub_intr_vec_count() - Interrupt Vectors Supported by a entity. + * ub_intr_vec_count() - Interrupt Vectors Supported by an entity. * @uent: UB entity. * - * Querying the Number of Interrupt Vectors Supported by a entity. + * Querying the Number of Interrupt Vectors Supported by an entity. * For interrupt type 2. * * Context: Any context. @@ -808,10 +807,10 @@ void ub_disable_intr(struct ub_entity *uent); u32 ub_intr_vec_count(struct ub_entity *uent); /** - * ub_int_type1_vec_count() - Interrupt Vectors Supported by a entity. + * ub_int_type1_vec_count() - Interrupt Vectors Supported by an entity. * @uent: UB entity. * - * Querying the Number of Interrupt Vectors Supported by a entity. + * Querying the Number of Interrupt Vectors Supported by an entity. * For interrupt type 1. * * Context: Any context. @@ -864,7 +863,7 @@ static inline int ub_alloc_irq_vectors(struct ub_entity *uent, int ub_irq_vector(struct ub_entity *uent, unsigned int nr); /** - * ub_irq_get_affinity() - Get a entity interrupt vector affinity + * ub_irq_get_affinity() - Get an entity interrupt vector affinity * @uent: the UB entity to operate on * @nr: entity-relative interrupt vector index (0-based); has different * meanings, depending on interrupt mode: @@ -884,7 +883,7 @@ const struct cpumask *ub_irq_get_affinity(struct ub_entity *uent, int nr); * @uent: UB entity. * @entity_idx: Number of the entity to be activated. * - * Context: Any context, It will take device_trylock()/device_unlock() + * Context: Any context, it will take device_trylock()/device_unlock() * Return: 0 if success, or %-EINVAL if the device doesn't match the driver, * or %-EBUSY if can't get device_trylock(), or other failed negative values. */ @@ -895,7 +894,7 @@ int ub_activate_entity(struct ub_entity *uent, u32 entity_idx); * @uent: UB entity. * @entity_idx: Number of the entity to be deactivated. * - * Context: Any context, It will take device_trylock()/device_unlock() + * Context: Any context, it will take device_trylock()/device_unlock() * Return: 0 if success, or %-EINVAL if the entity doesn't match the driver, * or %-EBUSY if can't get device_trylock(), or other failed negative values. */ @@ -910,7 +909,7 @@ int ub_deactivate_entity(struct ub_entity *uent, u32 entity_idx); * Initiate configuration access to the specified address of the entity * configuration space and read 1 byte. * - * Context: Any context, It will take spin_lock_irqsave()/spin_unlock_restore() + * Context: Any context, it will take spin_lock_irqsave()/spin_unlock_restore() * Return: 0 if success, or negative value if failed. */ int ub_cfg_read_byte(struct ub_entity *uent, u64 pos, u8 *val); @@ -925,7 +924,7 @@ int ub_cfg_read_dword(struct ub_entity *uent, u64 pos, u32 *val); * Initiate configuration access to the specified address of the entity * configuration space and write 1 byte. * - * Context: Any context, It will take spin_lock_irqsave()/spin_unlock_restore() + * Context: Any context, it will take spin_lock_irqsave()/spin_unlock_restore() * Return: 0 if success, or negative value if failed. */ int ub_cfg_write_byte(struct ub_entity *uent, u64 pos, u8 val); @@ -937,7 +936,7 @@ int ub_cfg_write_dword(struct ub_entity *uent, u64 pos, u32 val); * @uent: UB entity pointer. * * Context: Any context. - * Return: uent, or NULL if @uent is NULL. + * Return: @uent itself, or NULL if @uent is NULL. */ struct ub_entity *ub_entity_get(struct ub_entity *uent); @@ -955,9 +954,9 @@ void ub_entity_put(struct ub_entity *uent); * @max_num: Buffer size. * @real_num: Real entities num. * - * All ub bus controllers in the system are returned. Increase the reference - * counting of all entities by 1. Remember to call ub_put_bus_controller() after - * using it. + * All ub bus controllers in the system are collected in @uents. Increase the + * reference counting of all entities by 1. Remember to call + * ub_put_bus_controller() after using it. * * Context: Any context. * Return: 0 if success, or %-EINVAL if input parameter is NULL, @@ -1020,8 +1019,8 @@ void ub_unregister_driver(struct ub_driver *drv); * ub_stop_ent() - Stop the entity. * @uent: UB entity. * - * Call device_release_driver(), user can't use it again, if it's a mue, - * will stop all ues under it, if it's entity0, will stop all entity under it. + * Call device_release_driver(), user can't use it again. If it's a MUE, + * will stop all UEs under it. If it's entity0, will stop all entities under it. * * Context: Any context. */ @@ -1031,8 +1030,9 @@ void ub_stop_ent(struct ub_entity *uent); * ub_stop_and_remove_ent() - Stop and remove the entity from system. * @uent: UB entity. * - * Call device_release_driver() and device_unregister(), if it's a mue, - * will remove all ues under it, if it's entity0, will remove all entity under it. + * Call device_release_driver() and device_unregister(). If it's a MUE, + * will remove all UEs under it. If it's entity0, will remove all entities + * under it. * * Context: Any context. */ diff --git a/include/ub/urma/udma/udma_ctl.h b/include/ub/urma/udma/udma_ctl.h new file mode 100644 index 0000000000000000000000000000000000000000..19898d33c14bb7510b77674640754a9df310c97c --- /dev/null +++ b/include/ub/urma/udma/udma_ctl.h @@ -0,0 +1,213 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright(c) 2025 HiSilicon Technologies CO., Ltd. All rights reserved. */ + +#ifndef _UB_UMDK_URMA_UDMA_UDMA_CTL_H_ +#define _UB_UMDK_URMA_UDMA_UDMA_CTL_H_ + +#include + +#define UDMA_BUS_INSTANCE_SEID_SIZE 4 +#define UDMA_EID_PAIRS_COUNT 8 + +union udma_k_jfs_flag { + struct { + uint32_t sq_cstm : 1; + uint32_t db_cstm : 1; + uint32_t db_ctl_cstm : 1; + uint32_t reserved : 29; + } bs; + uint32_t value; +}; + +struct udma_que_cfg_ex { + uint32_t buff_size; + void *buff; +}; + +struct udma_jfs_cstm_cfg { + struct udma_que_cfg_ex sq; /* PA; should be converted by phys_to_virt. */ +}; + +struct udma_jfs_cfg_ex { + struct ubcore_jfs_cfg base_cfg; + struct ubcore_udata udata; + struct udma_jfs_cstm_cfg cstm_cfg; + ubcore_event_callback_t jfae_handler; +}; + +struct udma_jfc_cstm_cfg { + struct udma_que_cfg_ex cq; /* PA; should be using stars hw register addr. */ +}; + +struct udma_jfc_cfg_ex { + struct ubcore_jfc_cfg base_cfg; + struct ubcore_udata udata; + struct udma_jfc_cstm_cfg cstm_cfg; + ubcore_comp_callback_t jfce_handler; + ubcore_event_callback_t jfae_handler; +}; + +enum udma_jfc_type { + UDMA_NORMAL_JFC_TYPE, + UDMA_STARS_JFC_TYPE, + UDMA_CCU_JFC_TYPE, + UDMA_KERNEL_STARS_JFC_TYPE, + UDMA_JFC_TYPE_NUM, +}; + +struct udma_set_cqe_ex { + uint64_t addr; + uint32_t len; + enum udma_jfc_type jfc_type; +}; + +struct udma_ue_info_ex { + uint16_t ue_id; + uint32_t chip_id; + uint32_t die_id; + uint32_t offset_len; + resource_size_t db_base_addr; + resource_size_t dwqe_addr; + resource_size_t register_base_addr; +}; + +struct udma_tp_sport_in { + uint32_t tpn; +}; + +struct udma_tp_sport_out { + uint32_t data_udp_srcport; + uint32_t ack_udp_srcport; +}; + +struct udma_cqe_info_in { + enum ubcore_cr_status status; + uint8_t s_r; +}; + +enum udma_cqe_aux_info_type { + TPP2TQEM_WR_CNT, + DEVICE_RAS_STATUS_2, + RXDMA_WR_PAYL_AXI_ERR, + RXDMA_HEAD_SPLIT_ERR_FLAG0, + RXDMA_HEAD_SPLIT_ERR_FLAG1, + RXDMA_HEAD_SPLIT_ERR_FLAG2, + RXDMA_HEAD_SPLIT_ERR_FLAG3, + TP_RCP_INNER_ALM_FOR_CQE, + TWP_AE_DFX_FOR_CQE, + PA_OUT_PKT_ERR_CNT, + TP_DAM_AXI_ALARM, + TP_DAM_VFT_BT_ALARM, + TP_EUM_AXI_ALARM, + TP_EUM_VFT_BT_ALARM, + TP_TPMM_AXI_ALARM, + TP_TPMM_VFT_BT_ALARM, + TP_TPGCM_AXI_ALARM, + TP_TPGCM_VFT_BT_ALARM, + TWP_ALM, + TP_RWP_INNER_ALM_FOR_CQE, + TWP_DFX21, + LQC_TA_RNR_TANACK_CNT, + FVT, + RQMT0, + RQMT1, + RQMT2, + RQMT3, + RQMT4, + RQMT5, + RQMT6, + RQMT7, + RQMT8, + RQMT9, + RQMT10, + RQMT11, + RQMT12, + RQMT13, + RQMT14, + RQMT15, + PROC_ERROR_ALM, + LQC_TA_TIMEOUT_TAACK_CNT, + TP_RRP_ERR_FLG_0_FOR_CQE, + MAX_CQE_AUX_INFO_TYPE_NUM +}; + +struct udma_cqe_aux_info_out { + enum udma_cqe_aux_info_type *aux_info_type; + uint32_t *aux_info_value; + uint32_t aux_info_num; +}; + +struct udma_ae_info_in { + uint32_t event_type; +}; + +enum udma_ae_aux_info_type { + TP_RRP_FLUSH_TIMER_PKT_CNT, + TPP_DFX5, + TWP_AE_DFX_FOR_AE, + TP_RRP_ERR_FLG_0_FOR_AE, + TP_RRP_ERR_FLG_1, + TP_RWP_INNER_ALM_FOR_AE, + TP_RCP_INNER_ALM_FOR_AE, + LQC_TA_TQEP_WQE_ERR, + LQC_TA_CQM_CQE_INNER_ALARM, + MAX_AE_AUX_INFO_TYPE_NUM +}; + +struct udma_ae_aux_info_out { + enum udma_ae_aux_info_type *aux_info_type; + uint32_t *aux_info_value; + uint32_t aux_info_num; +}; + +enum udma_user_ctl_opcode { + UDMA_USER_CTL_CREATE_JFS_EX, + UDMA_USER_CTL_DELETE_JFS_EX, + UDMA_USER_CTL_CREATE_JFC_EX, + UDMA_USER_CTL_DELETE_JFC_EX, + UDMA_USER_CTL_SET_CQE_ADDR, + UDMA_USER_CTL_QUERY_UE_INFO, + UDMA_USER_CTL_GET_DEV_RES_RATIO, + UDMA_USER_CTL_NPU_REGISTER_INFO_CB, + UDMA_USER_CTL_NPU_UNREGISTER_INFO_CB, + UDMA_USER_CTL_QUERY_TP_SPORT, + UDMA_USER_CTL_QUERY_CQE_AUX_INFO, + UDMA_USER_CTL_QUERY_AE_AUX_INFO, + UDMA_USER_CTL_QUERY_UBMEM_INFO, + UDMA_USER_CTL_QUERY_PAIR_DEVNUM, + UDMA_USER_CTL_MAX, +}; + +struct udma_ctrlq_event_nb { + uint8_t opcode; + int (*crq_handler)(struct ubcore_device *dev, void *data, uint16_t len); +}; + +struct udma_dev_pair_info { + uint32_t peer_dev_id; + uint32_t slot_id; + uint32_t pair_num; + struct { + uint32_t local_eid[UDMA_BUS_INSTANCE_SEID_SIZE]; + uint32_t remote_eid[UDMA_BUS_INSTANCE_SEID_SIZE]; + uint32_t flag : 16; + uint32_t hop : 4; + uint32_t rsv : 12; + } eid_pairs[UDMA_EID_PAIRS_COUNT]; +}; + +static inline bool udma_check_base_param(uint64_t addr, uint32_t in_len, uint32_t len) +{ + return (addr == 0 || in_len != len); +} + +typedef int (*udma_user_ctl_ops)(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out); + +int udma_user_ctl(struct ubcore_device *dev, struct ubcore_user_ctl *k_user_ctl); +int udma_query_cqe_aux_info(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out); +int udma_query_ae_aux_info(struct ubcore_device *dev, struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out); + +#endif /* _UB_UMDK_URMA_UDMA_UDMA_CTL_H_ */ diff --git a/kernel/dma/map_benchmark.c b/kernel/dma/map_benchmark.c index cc19a3efea8960e325ad22cc37cff49a03842ec8..23aeba267cfac8ff9e5011a59247c833794f8227 100644 --- a/kernel/dma/map_benchmark.c +++ b/kernel/dma/map_benchmark.c @@ -18,6 +18,9 @@ #include #include #include +#ifdef CONFIG_UB_UBUS +#include +#endif struct map_benchmark_data { struct map_benchmark bparam; @@ -351,6 +354,19 @@ static struct pci_driver map_benchmark_pci_driver = { .probe = map_benchmark_pci_probe, }; +#ifdef CONFIG_UB_UBUS +static int map_benchmark_ub_probe(struct ub_entity *uent, + const struct ub_device_id *id) +{ + return __map_benchmark_probe(&uent->dev); +} + +static struct ub_driver map_benchmark_ub_driver = { + .name = "dma_map_benchmark", + .probe = map_benchmark_ub_probe, +}; +#endif + static int __init map_benchmark_init(void) { int ret; @@ -360,16 +376,30 @@ static int __init map_benchmark_init(void) return ret; ret = platform_driver_register(&map_benchmark_platform_driver); - if (ret) { - pci_unregister_driver(&map_benchmark_pci_driver); - return ret; - } + if (ret) + goto err_reg_platform; + +#ifdef CONFIG_UB_UBUS + ret = ub_register_driver(&map_benchmark_ub_driver); + if (ret) + goto err_reg_ub; +#endif return 0; +#ifdef CONFIG_UB_UBUS +err_reg_ub: + platform_driver_unregister(&map_benchmark_platform_driver); +#endif +err_reg_platform: + pci_unregister_driver(&map_benchmark_pci_driver); + return ret; } static void __exit map_benchmark_cleanup(void) { +#ifdef CONFIG_UB_UBUS + ub_unregister_driver(&map_benchmark_ub_driver); +#endif platform_driver_unregister(&map_benchmark_platform_driver); pci_unregister_driver(&map_benchmark_pci_driver); } diff --git a/kernel/panic.c b/kernel/panic.c index 8c54a4b96f03fe449a58653ba889e66263fe978c..0defd5bcfb89fa7fec3759e0c70fba0bcb047a46 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -482,6 +482,7 @@ const struct taint_flag taint_flags[TAINT_FLAGS_COUNT] = { [ TAINT_AUX ] = { 'X', ' ', true }, [ TAINT_RANDSTRUCT ] = { 'T', ' ', true }, [ TAINT_TEST ] = { 'N', ' ', true }, + [ TAINT_FWCTL ] = { 'J', ' ', true }, }; /** diff --git a/tools/debugging/kernel-chktaint b/tools/debugging/kernel-chktaint index 279be06332be990754490662ee00fa4257cecbda..e7da0909d09707c0a9b057c0a940772f57ed17eb 100755 --- a/tools/debugging/kernel-chktaint +++ b/tools/debugging/kernel-chktaint @@ -204,6 +204,14 @@ else echo " * an in-kernel test (such as a KUnit test) has been run (#18)" fi +T=`expr $T / 2` +if [ `expr $T % 2` -eq 0 ]; then + addout " " +else + addout "J" + echo " * fwctl's mutating debug interface was used (#19)" +fi + echo "For a more detailed explanation of the various taint flags see" echo " Documentation/admin-guide/tainted-kernels.rst in the Linux kernel sources" echo " or https://kernel.org/doc/html/latest/admin-guide/tainted-kernels.html"