diff options
705 files changed, 19119 insertions, 25973 deletions
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 184193bcb262..caf36105a1c7 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1857,8 +1857,10 @@ following two functions. wbc_init_bio(@wbc, @bio) Should be called for each bio carrying writeback data and - associates the bio with the inode's owner cgroup. Can be - called anytime between bio allocation and submission. + associates the bio with the inode's owner cgroup and the + corresponding request queue. This must be called after + a queue (device) has been associated with the bio and + before submission. wbc_account_io(@wbc, @page, @bytes) Should be called for each data segment being written out. @@ -1877,7 +1879,7 @@ the configuration, the bio may be executed at a lower priority and if the writeback session is holding shared resources, e.g. a journal entry, may lead to priority inversion. There is no one easy solution for the problem. Filesystems can try to work around specific problem -cases by skipping wbc_init_bio() or using bio_associate_blkcg() +cases by skipping wbc_init_bio() or using bio_associate_create_blkg() directly. diff --git a/Documentation/arm64/elf_hwcaps.txt b/Documentation/arm64/elf_hwcaps.txt index d6aff2c5e9e2..ea819ae024dd 100644 --- a/Documentation/arm64/elf_hwcaps.txt +++ b/Documentation/arm64/elf_hwcaps.txt @@ -78,11 +78,11 @@ HWCAP_EVTSTRM HWCAP_AES - Functionality implied by ID_AA64ISAR1_EL1.AES == 0b0001. + Functionality implied by ID_AA64ISAR0_EL1.AES == 0b0001. HWCAP_PMULL - Functionality implied by ID_AA64ISAR1_EL1.AES == 0b0010. + Functionality implied by ID_AA64ISAR0_EL1.AES == 0b0010. HWCAP_SHA1 @@ -153,7 +153,7 @@ HWCAP_ASIMDDP HWCAP_SHA512 - Functionality implied by ID_AA64ISAR0_EL1.SHA2 == 0b0002. + Functionality implied by ID_AA64ISAR0_EL1.SHA2 == 0b0010. HWCAP_SVE @@ -173,8 +173,12 @@ HWCAP_USCAT HWCAP_ILRCPC - Functionality implied by ID_AA64ISR1_EL1.LRCPC == 0b0002. + Functionality implied by ID_AA64ISAR1_EL1.LRCPC == 0b0010. HWCAP_FLAGM Functionality implied by ID_AA64ISAR0_EL1.TS == 0b0001. + +HWCAP_SSBS + + Functionality implied by ID_AA64PFR1_EL1.SSBS == 0b0010. diff --git a/Documentation/arm64/hugetlbpage.txt b/Documentation/arm64/hugetlbpage.txt new file mode 100644 index 000000000000..cfae87dc653b --- /dev/null +++ b/Documentation/arm64/hugetlbpage.txt @@ -0,0 +1,38 @@ +HugeTLBpage on ARM64 +==================== + +Hugepage relies on making efficient use of TLBs to improve performance of +address translations. The benefit depends on both - + + - the size of hugepages + - size of entries supported by the TLBs + +The ARM64 port supports two flavours of hugepages. + +1) Block mappings at the pud/pmd level +-------------------------------------- + +These are regular hugepages where a pmd or a pud page table entry points to a +block of memory. Regardless of the supported size of entries in TLB, block +mappings reduce the depth of page table walk needed to translate hugepage +addresses. + +2) Using the Contiguous bit +--------------------------- + +The architecture provides a contiguous bit in the translation table entries +(D4.5.3, ARM DDI 0487C.a) that hints to the MMU to indicate that it is one of a +contiguous set of entries that can be cached in a single TLB entry. + +The contiguous bit is used in Linux to increase the mapping size at the pmd and +pte (last) level. The number of supported contiguous entries varies by page size +and level of the page table. + + +The following hugepage sizes are supported - + + CONT PTE PMD CONT PMD PUD + -------- --- -------- --- + 4K: 64K 2M 32M 1G + 16K: 2M 32M 1G + 64K: 2M 512M 16G diff --git a/Documentation/arm64/silicon-errata.txt b/Documentation/arm64/silicon-errata.txt index 3b2f2dd82225..76ccded8b74c 100644 --- a/Documentation/arm64/silicon-errata.txt +++ b/Documentation/arm64/silicon-errata.txt @@ -56,6 +56,7 @@ stable kernels. | ARM | Cortex-A72 | #853709 | N/A | | ARM | Cortex-A73 | #858921 | ARM64_ERRATUM_858921 | | ARM | Cortex-A55 | #1024718 | ARM64_ERRATUM_1024718 | +| ARM | Cortex-A76 | #1188873 | ARM64_ERRATUM_1188873 | | ARM | MMU-500 | #841119,#826419 | N/A | | | | | | | Cavium | ThunderX ITS | #22375, #24313 | CAVIUM_ERRATUM_22375 | diff --git a/Documentation/blockdev/README.DAC960 b/Documentation/blockdev/README.DAC960 deleted file mode 100644 index bd85fb9dc6e5..000000000000 --- a/Documentation/blockdev/README.DAC960 +++ /dev/null @@ -1,756 +0,0 @@ - Linux Driver for Mylex DAC960/AcceleRAID/eXtremeRAID PCI RAID Controllers - - Version 2.2.11 for Linux 2.2.19 - Version 2.4.11 for Linux 2.4.12 - - PRODUCTION RELEASE - - 11 October 2001 - - Leonard N. Zubkoff - Dandelion Digital - lnz@dandelion.com - - Copyright 1998-2001 by Leonard N. Zubkoff <lnz@dandelion.com> - - - INTRODUCTION - -Mylex, Inc. designs and manufactures a variety of high performance PCI RAID -controllers. Mylex Corporation is located at 34551 Ardenwood Blvd., Fremont, -California 94555, USA and can be reached at 510.796.6100 or on the World Wide -Web at http://www.mylex.com. Mylex Technical Support can be reached by -electronic mail at mylexsup@us.ibm.com, by voice at 510.608.2400, or by FAX at -510.745.7715. Contact information for offices in Europe and Japan is available -on their Web site. - -The latest information on Linux support for DAC960 PCI RAID Controllers, as -well as the most recent release of this driver, will always be available from -my Linux Home Page at URL "http://www.dandelion.com/Linux/". The Linux DAC960 -driver supports all current Mylex PCI RAID controllers including the new -eXtremeRAID 2000/3000 and AcceleRAID 352/170/160 models which have an entirely -new firmware interface from the older eXtremeRAID 1100, AcceleRAID 150/200/250, -and DAC960PJ/PG/PU/PD/PL. See below for a complete controller list as well as -minimum firmware version requirements. For simplicity, in most places this -documentation refers to DAC960 generically rather than explicitly listing all -the supported models. - -Driver bug reports should be sent via electronic mail to "lnz@dandelion.com". -Please include with the bug report the complete configuration messages reported -by the driver at startup, along with any subsequent system messages relevant to -the controller's operation, and a detailed description of your system's -hardware configuration. Driver bugs are actually quite rare; if you encounter -problems with disks being marked offline, for example, please contact Mylex -Technical Support as the problem is related to the hardware configuration -rather than the Linux driver. - -Please consult the RAID controller documentation for detailed information -regarding installation and configuration of the controllers. This document -primarily provides information specific to the Linux support. - - - DRIVER FEATURES - -The DAC960 RAID controllers are supported solely as high performance RAID -controllers, not as interfaces to arbitrary SCSI devices. The Linux DAC960 -driver operates at the block device level, the same level as the SCSI and IDE -drivers. Unlike other RAID controllers currently supported on Linux, the -DAC960 driver is not dependent on the SCSI subsystem, and hence avoids all the -complexity and unnecessary code that would be associated with an implementation -as a SCSI driver. The DAC960 driver is designed for as high a performance as -possible with no compromises or extra code for compatibility with lower -performance devices. The DAC960 driver includes extensive error logging and -online configuration management capabilities. Except for initial configuration -of the controller and adding new disk drives, most everything can be handled -from Linux while the system is operational. - -The DAC960 driver is architected to support up to 8 controllers per system. -Each DAC960 parallel SCSI controller can support up to 15 disk drives per -channel, for a maximum of 60 drives on a four channel controller; the fibre -channel eXtremeRAID 3000 controller supports up to 125 disk drives per loop for -a total of 250 drives. The drives installed on a controller are divided into -one or more "Drive Groups", and then each Drive Group is subdivided further -into 1 to 32 "Logical Drives". Each Logical Drive has a specific RAID Level -and caching policy associated with it, and it appears to Linux as a single -block device. Logical Drives are further subdivided into up to 7 partitions -through the normal Linux and PC disk partitioning schemes. Logical Drives are -also known as "System Drives", and Drive Groups are also called "Packs". Both -terms are in use in the Mylex documentation; I have chosen to standardize on -the more generic "Logical Drive" and "Drive Group". - -DAC960 RAID disk devices are named in the style of the obsolete Device File -System (DEVFS). The device corresponding to Logical Drive D on Controller C -is referred to as /dev/rd/cCdD, and the partitions are called /dev/rd/cCdDp1 -through /dev/rd/cCdDp7. For example, partition 3 of Logical Drive 5 on -Controller 2 is referred to as /dev/rd/c2d5p3. Note that unlike with SCSI -disks the device names will not change in the event of a disk drive failure. -The DAC960 driver is assigned major numbers 48 - 55 with one major number per -controller. The 8 bits of minor number are divided into 5 bits for the Logical -Drive and 3 bits for the partition. - - - SUPPORTED DAC960/AcceleRAID/eXtremeRAID PCI RAID CONTROLLERS - -The following list comprises the supported DAC960, AcceleRAID, and eXtremeRAID -PCI RAID Controllers as of the date of this document. It is recommended that -anyone purchasing a Mylex PCI RAID Controller not in the following table -contact the author beforehand to verify that it is or will be supported. - -eXtremeRAID 3000 - 1 Wide Ultra-2/LVD SCSI channel - 2 External Fibre FC-AL channels - 233MHz StrongARM SA 110 Processor - 64 Bit 33MHz PCI (backward compatible with 32 Bit PCI slots) - 32MB/64MB ECC SDRAM Memory - -eXtremeRAID 2000 - 4 Wide Ultra-160 LVD SCSI channels - 233MHz StrongARM SA 110 Processor - 64 Bit 33MHz PCI (backward compatible with 32 Bit PCI slots) - 32MB/64MB ECC SDRAM Memory - -AcceleRAID 352 - 2 Wide Ultra-160 LVD SCSI channels - 100MHz Intel i960RN RISC Processor - 64 Bit 33MHz PCI (backward compatible with 32 Bit PCI slots) - 32MB/64MB ECC SDRAM Memory - -AcceleRAID 170 - 1 Wide Ultra-160 LVD SCSI channel - 100MHz Intel i960RM RISC Processor - 16MB/32MB/64MB ECC SDRAM Memory - -AcceleRAID 160 (AcceleRAID 170LP) - 1 Wide Ultra-160 LVD SCSI channel - 100MHz Intel i960RS RISC Processor - Built in 16M ECC SDRAM Memory - PCI Low Profile Form Factor - fit for 2U height - -eXtremeRAID 1100 (DAC1164P) - 3 Wide Ultra-2/LVD SCSI channels - 233MHz StrongARM SA 110 Processor - 64 Bit 33MHz PCI (backward compatible with 32 Bit PCI slots) - 16MB/32MB/64MB Parity SDRAM Memory with Battery Backup - -AcceleRAID 250 (DAC960PTL1) - Uses onboard Symbios SCSI chips on certain motherboards - Also includes one onboard Wide Ultra-2/LVD SCSI Channel - 66MHz Intel i960RD RISC Processor - 4MB/8MB/16MB/32MB/64MB/128MB ECC EDO Memory - -AcceleRAID 200 (DAC960PTL0) - Uses onboard Symbios SCSI chips on certain motherboards - Includes no onboard SCSI Channels - 66MHz Intel i960RD RISC Processor - 4MB/8MB/16MB/32MB/64MB/128MB ECC EDO Memory - -AcceleRAID 150 (DAC960PRL) - Uses onboard Symbios SCSI chips on certain motherboards - Also includes one onboard Wide Ultra-2/LVD SCSI Channel - 33MHz Intel i960RP RISC Processor - 4MB Parity EDO Memory - -DAC960PJ 1/2/3 Wide Ultra SCSI-3 Channels - 66MHz Intel i960RD RISC Processor - 4MB/8MB/16MB/32MB/64MB/128MB ECC EDO Memory - -DAC960PG 1/2/3 Wide Ultra SCSI-3 Channels - 33MHz Intel i960RP RISC Processor - 4MB/8MB ECC EDO Memory - -DAC960PU 1/2/3 Wide Ultra SCSI-3 Channels - Intel i960CF RISC Processor - 4MB/8MB EDRAM or 2MB/4MB/8MB/16MB/32MB DRAM Memory - -DAC960PD 1/2/3 Wide Fast SCSI-2 Channels - Intel i960CF RISC Processor - 4MB/8MB EDRAM or 2MB/4MB/8MB/16MB/32MB DRAM Memory - -DAC960PL 1/2/3 Wide Fast SCSI-2 Channels - Intel i960 RISC Processor - 2MB/4MB/8MB/16MB/32MB DRAM Memory - -DAC960P 1/2/3 Wide Fast SCSI-2 Channels - Intel i960 RISC Processor - 2MB/4MB/8MB/16MB/32MB DRAM Memory - -For the eXtremeRAID 2000/3000 and AcceleRAID 352/170/160, firmware version -6.00-01 or above is required. - -For the eXtremeRAID 1100, firmware version 5.06-0-52 or above is required. - -For the AcceleRAID 250, 200, and 150, firmware version 4.06-0-57 or above is -required. - -For the DAC960PJ and DAC960PG, firmware version 4.06-0-00 or above is required. - -For the DAC960PU, DAC960PD, DAC960PL, and DAC960P, either firmware version -3.51-0-04 or above is required (for dual Flash ROM controllers), or firmware -version 2.73-0-00 or above is required (for single Flash ROM controllers) - -Please note that not all SCSI disk drives are suitable for use with DAC960 -controllers, and only particular firmware versions of any given model may -actually function correctly. Similarly, not all motherboards have a BIOS that -properly initializes the AcceleRAID 250, AcceleRAID 200, AcceleRAID 150, -DAC960PJ, and DAC960PG because the Intel i960RD/RP is a multi-function device. -If in doubt, contact Mylex RAID Technical Support (mylexsup@us.ibm.com) to -verify compatibility. Mylex makes available a hard disk compatibility list at -http://www.mylex.com/support/hdcomp/hd-lists.html. - - - DRIVER INSTALLATION - -This distribution was prepared for Linux kernel version 2.2.19 or 2.4.12. - -To install the DAC960 RAID driver, you may use the following commands, -replacing "/usr/src" with wherever you keep your Linux kernel source tree: - - cd /usr/src - tar -xvzf DAC960-2.2.11.tar.gz (or DAC960-2.4.11.tar.gz) - mv README.DAC960 linux/Documentation - mv DAC960.[ch] linux/drivers/block - patch -p0 < DAC960.patch (if DAC960.patch is included) - cd linux - make config - make bzImage (or zImage) - -Then install "arch/x86/boot/bzImage" or "arch/x86/boot/zImage" as your -standard kernel, run lilo if appropriate, and reboot. - -To create the necessary devices in /dev, the "make_rd" script included in -"DAC960-Utilities.tar.gz" from http://www.dandelion.com/Linux/ may be used. -LILO 21 and FDISK v2.9 include DAC960 support; also included in this archive -are patches to LILO 20 and FDISK v2.8 that add DAC960 support, along with -statically linked executables of LILO and FDISK. This modified version of LILO -will allow booting from a DAC960 controller and/or mounting the root file -system from a DAC960. - -Red Hat Linux 6.0 and SuSE Linux 6.1 include support for Mylex PCI RAID -controllers. Installing directly onto a DAC960 may be problematic from other -Linux distributions until their installation utilities are updated. - - - INSTALLATION NOTES - -Before installing Linux or adding DAC960 logical drives to an existing Linux -system, the controller must first be configured to provide one or more logical -drives using the BIOS Configuration Utility or DACCF. Please note that since -there are only at most 6 usable partitions on each logical drive, systems -requiring more partitions should subdivide a drive group into multiple logical -drives, each of which can have up to 6 usable partitions. Also, note that with -large disk arrays it is advisable to enable the 8GB BIOS Geometry (255/63) -rather than accepting the default 2GB BIOS Geometry (128/32); failing to so do -will cause the logical drive geometry to have more than 65535 cylinders which -will make it impossible for FDISK to be used properly. The 8GB BIOS Geometry -can be enabled by configuring the DAC960 BIOS, which is accessible via Alt-M -during the BIOS initialization sequence. - -For maximum performance and the most efficient E2FSCK performance, it is -recommended that EXT2 file systems be built with a 4KB block size and 16 block -stride to match the DAC960 controller's 64KB default stripe size. The command -"mke2fs -b 4096 -R stride=16 <device>" is appropriate. Unless there will be a -large number of small files on the file systems, it is also beneficial to add -the "-i 16384" option to increase the bytes per inode parameter thereby -reducing the file system metadata. Finally, on systems that will only be run -with Linux 2.2 or later kernels it is beneficial to enable sparse superblocks -with the "-s 1" option. - - - DAC960 ANNOUNCEMENTS MAILING LIST - -The DAC960 Announcements Mailing List provides a forum for informing Linux -users of new driver releases and other announcements regarding Linux support -for DAC960 PCI RAID Controllers. To join the mailing list, send a message to -"dac960-announce-request@dandelion.com" with the line "subscribe" in the -message body. - - - CONTROLLER CONFIGURATION AND STATUS MONITORING - -The DAC960 RAID controllers running firmware 4.06 or above include a Background -Initialization facility so that system downtime is minimized both for initial -installation and subsequent configuration of additional storage. The BIOS -Configuration Utility (accessible via Alt-R during the BIOS initialization -sequence) is used to quickly configure the controller, and then the logical -drives that have been created are available for immediate use even while they -are still being initialized by the controller. The primary need for online -configuration and status monitoring is then to avoid system downtime when disk -drives fail and must be replaced. Mylex's online monitoring and configuration -utilities are being ported to Linux and will become available at some point in -the future. Note that with a SAF-TE (SCSI Accessed Fault-Tolerant Enclosure) -enclosure, the controller is able to rebuild failed drives automatically as -soon as a drive replacement is made available. - -The primary interfaces for controller configuration and status monitoring are -special files created in the /proc/rd/... hierarchy along with the normal -system console logging mechanism. Whenever the system is operating, the DAC960 -driver queries each controller for status information every 10 seconds, and -checks for additional conditions every 60 seconds. The initial status of each -controller is always available for controller N in /proc/rd/cN/initial_status, -and the current status as of the last status monitoring query is available in -/proc/rd/cN/current_status. In addition, status changes are also logged by the -driver to the system console and will appear in the log files maintained by -syslog. The progress of asynchronous rebuild or consistency check operations -is also available in /proc/rd/cN/current_status, and progress messages are -logged to the system console at most every 60 seconds. - -Starting with the 2.2.3/2.0.3 versions of the driver, the status information -available in /proc/rd/cN/initial_status and /proc/rd/cN/current_status has been -augmented to include the vendor, model, revision, and serial number (if -available) for each physical device found connected to the controller: - -***** DAC960 RAID Driver Version 2.2.3 of 19 August 1999 ***** -Copyright 1998-1999 by Leonard N. Zubkoff <lnz@dandelion.com> -Configuring Mylex DAC960PRL PCI RAID Controller - Firmware Version: 4.07-0-07, Channels: 1, Memory Size: 16MB - PCI Bus: 1, Device: 4, Function: 1, I/O Address: Unassigned - PCI Address: 0xFE300000 mapped at 0xA0800000, IRQ Channel: 21 - Controller Queue Depth: 128, Maximum Blocks per Command: 128 - Driver Queue Depth: 127, Maximum Scatter/Gather Segments: 33 - Stripe Size: 64KB, Segment Size: 8KB, BIOS Geometry: 255/63 - SAF-TE Enclosure Management Enabled - Physical Devices: - 0:0 Vendor: IBM Model: DRVS09D Revision: 0270 - Serial Number: 68016775HA - Disk Status: Online, 17928192 blocks - 0:1 Vendor: IBM Model: DRVS09D Revision: 0270 - Serial Number: 68004E53HA - Disk Status: Online, 17928192 blocks - 0:2 Vendor: IBM Model: DRVS09D Revision: 0270 - Serial Number: 13013935HA - Disk Status: Online, 17928192 blocks - 0:3 Vendor: IBM Model: DRVS09D Revision: 0270 - Serial Number: 13016897HA - Disk Status: Online, 17928192 blocks - 0:4 Vendor: IBM Model: DRVS09D Revision: 0270 - Serial Number: 68019905HA - Disk Status: Online, 17928192 blocks - 0:5 Vendor: IBM Model: DRVS09D Revision: 0270 - Serial Number: 68012753HA - Disk Status: Online, 17928192 blocks - 0:6 Vendor: ESG-SHV Model: SCA HSBP M6 Revision: 0.61 - Logical Drives: - /dev/rd/c0d0: RAID-5, Online, 89640960 blocks, Write Thru - No Rebuild or Consistency Check in Progress - -To simplify the monitoring process for custom software, the special file -/proc/rd/status returns "OK" when all DAC960 controllers in the system are -operating normally and no failures have occurred, or "ALERT" if any logical -drives are offline or critical or any non-standby physical drives are dead. - -Configuration commands for controller N are available via the special file -/proc/rd/cN/user_command. A human readable command can be written to this -special file to initiate a configuration operation, and the results of the -operation can then be read back from the special file in addition to being -logged to the system console. The shell command sequence - - echo "<configuration-command>" > /proc/rd/c0/user_command - cat /proc/rd/c0/user_command - -is typically used to execute configuration commands. The configuration -commands are: - - flush-cache - - The "flush-cache" command flushes the controller's cache. The system - automatically flushes the cache at shutdown or if the driver module is - unloaded, so this command is only needed to be certain a write back cache - is flushed to disk before the system is powered off by a command to a UPS. - Note that the flush-cache command also stops an asynchronous rebuild or - consistency check, so it should not be used except when the system is being - halted. - - kill <channel>:<target-id> - - The "kill" command marks the physical drive <channel>:<target-id> as DEAD. - This command is provided primarily for testing, and should not be used - during normal system operation. - - make-online <channel>:<target-id> - - The "make-online" command changes the physical drive <channel>:<target-id> - from status DEAD to status ONLINE. In cases where multiple physical drives - have been killed simultaneously, this command may be used to bring all but - one of them back online, after which a rebuild to the final drive is - necessary. - - Warning: make-online should only be used on a dead physical drive that is - an active part of a drive group, never on a standby drive. The command - should never be used on a dead drive that is part of a critical logical - drive; rebuild should be used if only a single drive is dead. - - make-standby <channel>:<target-id> - - The "make-standby" command changes physical drive <channel>:<target-id> - from status DEAD to status STANDBY. It should only be used in cases where - a dead drive was replaced after an automatic rebuild was performed onto a - standby drive. It cannot be used to add a standby drive to the controller - configuration if one was not created initially; the BIOS Configuration - Utility must be used for that currently. - - rebuild <channel>:<target-id> - - The "rebuild" command initiates an asynchronous rebuild onto physical drive - <channel>:<target-id>. It should only be used when a dead drive has been - replaced. - - check-consistency <logical-drive-number> - - The "check-consistency" command initiates an asynchronous consistency check - of <logical-drive-number> with automatic restoration. It can be used - whenever it is desired to verify the consistency of the redundancy - information. - - cancel-rebuild - cancel-consistency-check - - The "cancel-rebuild" and "cancel-consistency-check" commands cancel any - rebuild or consistency check operations previously initiated. - - - EXAMPLE I - DRIVE FAILURE WITHOUT A STANDBY DRIVE - -The following annotated logs demonstrate the controller configuration and and -online status monitoring capabilities of the Linux DAC960 Driver. The test -configuration comprises 6 1GB Quantum Atlas I disk drives on two channels of a -DAC960PJ controller. The physical drives are configured into a single drive -group without a standby drive, and the drive group has been configured into two -logical drives, one RAID-5 and one RAID-6. Note that these logs are from an -earlier version of the driver and the messages have changed somewhat with newer -releases, but the functionality remains similar. First, here is the current -status of the RAID configuration: - -gwynedd:/u/lnz# cat /proc/rd/c0/current_status -***** DAC960 RAID Driver Version 2.0.0 of 23 March 1999 ***** -Copyright 1998-1999 by Leonard N. Zubkoff <lnz@dandelion.com> -Configuring Mylex DAC960PJ PCI RAID Controller - Firmware Version: 4.06-0-08, Channels: 3, Memory Size: 8MB - PCI Bus: 0, Device: 19, Function: 1, I/O Address: Unassigned - PCI Address: 0xFD4FC000 mapped at 0x8807000, IRQ Channel: 9 - Controller Queue Depth: 128, Maximum Blocks per Command: 128 - Driver Queue Depth: 127, Maximum Scatter/Gather Segments: 33 - Stripe Size: 64KB, Segment Size: 8KB, BIOS Geometry: 255/63 - Physical Devices: - 0:1 - Disk: Online, 2201600 blocks - 0:2 - Disk: Online, 2201600 blocks - 0:3 - Disk: Online, 2201600 blocks - 1:1 - Disk: Online, 2201600 blocks - 1:2 - Disk: Online, 2201600 blocks - 1:3 - Disk: Online, 2201600 blocks - Logical Drives: - /dev/rd/c0d0: RAID-5, Online, 5498880 blocks, Write Thru - /dev/rd/c0d1: RAID-6, Online, 3305472 blocks, Write Thru - No Rebuild or Consistency Check in Progress - -gwynedd:/u/lnz# cat /proc/rd/status -OK - -The above messages indicate that everything is healthy, and /proc/rd/status -returns "OK" indicating that there are no problems with any DAC960 controller -in the system. For demonstration purposes, while I/O is active Physical Drive -1:1 is now disconnected, simulating a drive failure. The failure is noted by -the driver within 10 seconds of the controller's having detected it, and the -driver logs the following console status messages indicating that Logical -Drives 0 and 1 are now CRITICAL as a result of Physical Drive 1:1 being DEAD: - -DAC960#0: Physical Drive 1:2 Error Log: Sense Key = 6, ASC = 29, ASCQ = 02 -DAC960#0: Physical Drive 1:3 Error Log: Sense Key = 6, ASC = 29, ASCQ = 02 -DAC960#0: Physical Drive 1:1 killed because of timeout on SCSI command -DAC960#0: Physical Drive 1:1 is now DEAD -DAC960#0: Logical Drive 0 (/dev/rd/c0d0) is now CRITICAL -DAC960#0: Logical Drive 1 (/dev/rd/c0d1) is now CRITICAL - -The Sense Keys logged here are just Check Condition / Unit Attention conditions -arising from a SCSI bus reset that is forced by the controller during its error -recovery procedures. Concurrently with the above, the driver status available -from /proc/rd also reflects the drive failure. The status message in -/proc/rd/status has changed from "OK" to "ALERT": - -gwynedd:/u/lnz# cat /proc/rd/status -ALERT - -and /proc/rd/c0/current_status has been updated: - -gwynedd:/u/lnz# cat /proc/rd/c0/current_status - ... - Physical Devices: - 0:1 - Disk: Online, 2201600 blocks - 0:2 - Disk: Online, 2201600 blocks - 0:3 - Disk: Online, 2201600 blocks - 1:1 - Disk: Dead, 2201600 blocks - 1:2 - Disk: Online, 2201600 blocks - 1:3 - Disk: Online, 2201600 blocks - Logical Drives: - /dev/rd/c0d0: RAID-5, Critical, 5498880 blocks, Write Thru - /dev/rd/c0d1: RAID-6, Critical, 3305472 blocks, Write Thru - No Rebuild or Consistency Check in Progress - -Since there are no standby drives configured, the system can continue to access -the logical drives in a performance degraded mode until the failed drive is -replaced and a rebuild operation completed to restore the redundancy of the -logical drives. Once Physical Drive 1:1 is replaced with a properly -functioning drive, or if the physical drive was killed without having failed -(e.g., due to electrical problems on the SCSI bus), the user can instruct the -controller to initiate a rebuild operation onto the newly replaced drive: - -gwynedd:/u/lnz# echo "rebuild 1:1" > /proc/rd/c0/user_command -gwynedd:/u/lnz# cat /proc/rd/c0/user_command -Rebuild of Physical Drive 1:1 Initiated - -The echo command instructs the controller to initiate an asynchronous rebuild -operation onto Physical Drive 1:1, and the status message that results from the -operation is then available for reading from /proc/rd/c0/user_command, as well -as being logged to the console by the driver. - -Within 10 seconds of this command the driver logs the initiation of the -asynchronous rebuild operation: - -DAC960#0: Rebuild of Physical Drive 1:1 Initiated -DAC960#0: Physical Drive 1:1 Error Log: Sense Key = 6, ASC = 29, ASCQ = 01 -DAC960#0: Physical Drive 1:1 is now WRITE-ONLY -DAC960#0: Rebuild in Progress: Logical Drive 0 (/dev/rd/c0d0) 1% completed - -and /proc/rd/c0/current_status is updated: - -gwynedd:/u/lnz# cat /proc/rd/c0/current_status - ... - Physical Devices: - 0:1 - Disk: Online, 2201600 blocks - 0:2 - Disk: Online, 2201600 blocks - 0:3 - Disk: Online, 2201600 blocks - 1:1 - Disk: Write-Only, 2201600 blocks - 1:2 - Disk: Online, 2201600 blocks - 1:3 - Disk: Online, 2201600 blocks - Logical Drives: - /dev/rd/c0d0: RAID-5, Critical, 5498880 blocks, Write Thru - /dev/rd/c0d1: RAID-6, Critical, 3305472 blocks, Write Thru - Rebuild in Progress: Logical Drive 0 (/dev/rd/c0d0) 6% completed - -As the rebuild progresses, the current status in /proc/rd/c0/current_status is -updated every 10 seconds: - -gwynedd:/u/lnz# cat /proc/rd/c0/current_status - ... - Physical Devices: - 0:1 - Disk: Online, 2201600 blocks - 0:2 - Disk: Online, 2201600 blocks - 0:3 - Disk: Online, 2201600 blocks - 1:1 - Disk: Write-Only, 2201600 blocks - 1:2 - Disk: Online, 2201600 blocks - 1:3 - Disk: Online, 2201600 blocks - Logical Drives: - /dev/rd/c0d0: RAID-5, Critical, 5498880 blocks, Write Thru - /dev/rd/c0d1: RAID-6, Critical, 3305472 blocks, Write Thru - Rebuild in Progress: Logical Drive 0 (/dev/rd/c0d0) 15% completed - -and every minute a progress message is logged to the console by the driver: - -DAC960#0: Rebuild in Progress: Logical Drive 0 (/dev/rd/c0d0) 32% completed -DAC960#0: Rebuild in Progress: Logical Drive 0 (/dev/rd/c0d0) 63% completed -DAC960#0: Rebuild in Progress: Logical Drive 0 (/dev/rd/c0d0) 94% completed -DAC960#0: Rebuild in Progress: Logical Drive 1 (/dev/rd/c0d1) 94% completed - -Finally, the rebuild completes successfully. The driver logs the status of the -logical and physical drives and the rebuild completion: - -DAC960#0: Rebuild Completed Successfully -DAC960#0: Physical Drive 1:1 is now ONLINE -DAC960#0: Logical Drive 0 (/dev/rd/c0d0) is now ONLINE -DAC960#0: Logical Drive 1 (/dev/rd/c0d1) is now ONLINE - -/proc/rd/c0/current_status is updated: - -gwynedd:/u/lnz# cat /proc/rd/c0/current_status - ... - Physical Devices: - 0:1 - Disk: Online, 2201600 blocks - 0:2 - Disk: Online, 2201600 blocks - 0:3 - Disk: Online, 2201600 blocks - 1:1 - Disk: Online, 2201600 blocks - 1:2 - Disk: Online, 2201600 blocks - 1:3 - Disk: Online, 2201600 blocks - Logical Drives: - /dev/rd/c0d0: RAID-5, Online, 5498880 blocks, Write Thru - /dev/rd/c0d1: RAID-6, Online, 3305472 blocks, Write Thru - Rebuild Completed Successfully - -and /proc/rd/status indicates that everything is healthy once again: - -gwynedd:/u/lnz# cat /proc/rd/status -OK - - - EXAMPLE II - DRIVE FAILURE WITH A STANDBY DRIVE - -The following annotated logs demonstrate the controller configuration and and -online status monitoring capabilities of the Linux DAC960 Driver. The test -configuration comprises 6 1GB Quantum Atlas I disk drives on two channels of a -DAC960PJ controller. The physical drives are configured into a single drive -group with a standby drive, and the drive group has been configured into two -logical drives, one RAID-5 and one RAID-6. Note that these logs are from an -earlier version of the driver and the messages have changed somewhat with newer -releases, but the functionality remains similar. First, here is the current -status of the RAID configuration: - -gwynedd:/u/lnz# cat /proc/rd/c0/current_status -***** DAC960 RAID Driver Version 2.0.0 of 23 March 1999 ***** -Copyright 1998-1999 by Leonard N. Zubkoff <lnz@dandelion.com> -Configuring Mylex DAC960PJ PCI RAID Controller - Firmware Version: 4.06-0-08, Channels: 3, Memory Size: 8MB - PCI Bus: 0, Device: 19, Function: 1, I/O Address: Unassigned - PCI Address: 0xFD4FC000 mapped at 0x8807000, IRQ Channel: 9 - Controller Queue Depth: 128, Maximum Blocks per Command: 128 - Driver Queue Depth: 127, Maximum Scatter/Gather Segments: 33 - Stripe Size: 64KB, Segment Size: 8KB, BIOS Geometry: 255/63 - Physical Devices: - 0:1 - Disk: Online, 2201600 blocks - 0:2 - Disk: Online, 2201600 blocks - 0:3 - Disk: Online, 2201600 blocks - 1:1 - Disk: Online, 2201600 blocks - 1:2 - Disk: Online, 2201600 blocks - 1:3 - Disk: Standby, 2201600 blocks - Logical Drives: - /dev/rd/c0d0: RAID-5, Online, 4399104 blocks, Write Thru - /dev/rd/c0d1: RAID-6, Online, 2754560 blocks, Write Thru - No Rebuild or Consistency Check in Progress - -gwynedd:/u/lnz# cat /proc/rd/status -OK - -The above messages indicate that everything is healthy, and /proc/rd/status -returns "OK" indicating that there are no problems with any DAC960 controller -in the system. For demonstration purposes, while I/O is active Physical Drive -1:2 is now disconnected, simulating a drive failure. The failure is noted by -the driver within 10 seconds of the controller's having detected it, and the -driver logs the following console status messages: - -DAC960#0: Physical Drive 1:1 Error Log: Sense Key = 6, ASC = 29, ASCQ = 02 -DAC960#0: Physical Drive 1:3 Error Log: Sense Key = 6, ASC = 29, ASCQ = 02 -DAC960#0: Physical Drive 1:2 killed because of timeout on SCSI command -DAC960#0: Physical Drive 1:2 is now DEAD -DAC960#0: Physical Drive 1:2 killed because it was removed -DAC960#0: Logical Drive 0 (/dev/rd/c0d0) is now CRITICAL -DAC960#0: Logical Drive 1 (/dev/rd/c0d1) is now CRITICAL - -Since a standby drive is configured, the controller automatically begins -rebuilding onto the standby drive: - -DAC960#0: Physical Drive 1:3 is now WRITE-ONLY -DAC960#0: Rebuild in Progress: Logical Drive 0 (/dev/rd/c0d0) 4% completed - -Concurrently with the above, the driver status available from /proc/rd also -reflects the drive failure and automatic rebuild. The status message in -/proc/rd/status has changed from "OK" to "ALERT": - -gwynedd:/u/lnz# cat /proc/rd/status -ALERT - -and /proc/rd/c0/current_status has been updated: - -gwynedd:/u/lnz# cat /proc/rd/c0/current_status - ... - Physical Devices: - 0:1 - Disk: Online, 2201600 blocks - 0:2 - Disk: Online, 2201600 blocks - 0:3 - Disk: Online, 2201600 blocks - 1:1 - Disk: Online, 2201600 blocks - 1:2 - Disk: Dead, 2201600 blocks - 1:3 - Disk: Write-Only, 2201600 blocks - Logical Drives: - /dev/rd/c0d0: RAID-5, Critical, 4399104 blocks, Write Thru - /dev/rd/c0d1: RAID-6, Critical, 2754560 blocks, Write Thru - Rebuild in Progress: Logical Drive 0 (/dev/rd/c0d0) 4% completed - -As the rebuild progresses, the current status in /proc/rd/c0/current_status is -updated every 10 seconds: - -gwynedd:/u/lnz# cat /proc/rd/c0/current_status - ... - Physical Devices: - 0:1 - Disk: Online, 2201600 blocks - 0:2 - Disk: Online, 2201600 blocks - 0:3 - Disk: Online, 2201600 blocks - 1:1 - Disk: Online, 2201600 blocks - 1:2 - Disk: Dead, 2201600 blocks - 1:3 - Disk: Write-Only, 2201600 blocks - Logical Drives: - /dev/rd/c0d0: RAID-5, Critical, 4399104 blocks, Write Thru - /dev/rd/c0d1: RAID-6, Critical, 2754560 blocks, Write Thru - Rebuild in Progress: Logical Drive 0 (/dev/rd/c0d0) 40% completed - -and every minute a progress message is logged on the console by the driver: - -DAC960#0: Rebuild in Progress: Logical Drive 0 (/dev/rd/c0d0) 40% completed -DAC960#0: Rebuild in Progress: Logical Drive 0 (/dev/rd/c0d0) 76% completed -DAC960#0: Rebuild in Progress: Logical Drive 1 (/dev/rd/c0d1) 66% completed -DAC960#0: Rebuild in Progress: Logical Drive 1 (/dev/rd/c0d1) 84% completed - -Finally, the rebuild completes successfully. The driver logs the status of the -logical and physical drives and the rebuild completion: - -DAC960#0: Rebuild Completed Successfully -DAC960#0: Physical Drive 1:3 is now ONLINE -DAC960#0: Logical Drive 0 (/dev/rd/c0d0) is now ONLINE -DAC960#0: Logical Drive 1 (/dev/rd/c0d1) is now ONLINE - -/proc/rd/c0/current_status is updated: - -***** DAC960 RAID Driver Version 2.0.0 of 23 March 1999 ***** -Copyright 1998-1999 by Leonard N. Zubkoff <lnz@dandelion.com> -Configuring Mylex DAC960PJ PCI RAID Controller - Firmware Version: 4.06-0-08, Channels: 3, Memory Size: 8MB - PCI Bus: 0, Device: 19, Function: 1, I/O Address: Unassigned - PCI Address: 0xFD4FC000 mapped at 0x8807000, IRQ Channel: 9 - Controller Queue Depth: 128, Maximum Blocks per Command: 128 - Driver Queue Depth: 127, Maximum Scatter/Gather Segments: 33 - Stripe Size: 64KB, Segment Size: 8KB, BIOS Geometry: 255/63 - Physical Devices: - 0:1 - Disk: Online, 2201600 blocks - 0:2 - Disk: Online, 2201600 blocks - 0:3 - Disk: Online, 2201600 blocks - 1:1 - Disk: Online, 2201600 blocks - 1:2 - Disk: Dead, 2201600 blocks - 1:3 - Disk: Online, 2201600 blocks - Logical Drives: - /dev/rd/c0d0: RAID-5, Online, 4399104 blocks, Write Thru - /dev/rd/c0d1: RAID-6, Online, 2754560 blocks, Write Thru - Rebuild Completed Successfully - -and /proc/rd/status indicates that everything is healthy once again: - -gwynedd:/u/lnz# cat /proc/rd/status -OK - -Note that the absence of a viable standby drive does not create an "ALERT" -status. Once dead Physical Drive 1:2 has been replaced, the controller must be -told that this has occurred and that the newly replaced drive should become the -new standby drive: - -gwynedd:/u/lnz# echo "make-standby 1:2" > /proc/rd/c0/user_command -gwynedd:/u/lnz# cat /proc/rd/c0/user_command -Make Standby of Physical Drive 1:2 Succeeded - -The echo command instructs the controller to make Physical Drive 1:2 into a -standby drive, and the status message that results from the operation is then -available for reading from /proc/rd/c0/user_command, as well as being logged to -the console by the driver. Within 60 seconds of this command the driver logs: - -DAC960#0: Physical Drive 1:2 Error Log: Sense Key = 6, ASC = 29, ASCQ = 01 -DAC960#0: Physical Drive 1:2 is now STANDBY -DAC960#0: Make Standby of Physical Drive 1:2 Succeeded - -and /proc/rd/c0/current_status is updated: - -gwynedd:/u/lnz# cat /proc/rd/c0/current_status - ... - Physical Devices: - 0:1 - Disk: Online, 2201600 blocks - 0:2 - Disk: Online, 2201600 blocks - 0:3 - Disk: Online, 2201600 blocks - 1:1 - Disk: Online, 2201600 blocks - 1:2 - Disk: Standby, 2201600 blocks - 1:3 - Disk: Online, 2201600 blocks - Logical Drives: - /dev/rd/c0d0: RAID-5, Online, 4399104 blocks, Write Thru - /dev/rd/c0d1: RAID-6, Online, 2754560 blocks, Write Thru - Rebuild Completed Successfully diff --git a/Documentation/blockdev/zram.txt b/Documentation/blockdev/zram.txt index 875b2b56b87f..3c1b5ab54bc0 100644 --- a/Documentation/blockdev/zram.txt +++ b/Documentation/blockdev/zram.txt @@ -190,7 +190,7 @@ whitespace: notify_free Depending on device usage scenario it may account a) the number of pages freed because of swap slot free notifications or b) the number of pages freed because of - REQ_DISCARD requests sent by bio. The former ones are + REQ_OP_DISCARD requests sent by bio. The former ones are sent to a swap block device when a swap slot is freed, which implies that this disk is being used as a swap disk. The latter ones are sent by filesystem mounted with diff --git a/Documentation/core-api/idr.rst b/Documentation/core-api/idr.rst index d351e880a2f6..a2738050c4f0 100644 --- a/Documentation/core-api/idr.rst +++ b/Documentation/core-api/idr.rst @@ -1,4 +1,4 @@ -.. SPDX-License-Identifier: CC-BY-SA-4.0 +.. SPDX-License-Identifier: GPL-2.0+ ============= ID Allocation diff --git a/Documentation/device-mapper/log-writes.txt b/Documentation/device-mapper/log-writes.txt index f4ebcbaf50f3..b638d124be6a 100644 --- a/Documentation/device-mapper/log-writes.txt +++ b/Documentation/device-mapper/log-writes.txt @@ -38,7 +38,7 @@ inconsistent file system. Any REQ_FUA requests bypass this flushing mechanism and are logged as soon as they complete as those requests will obviously bypass the device cache. -Any REQ_DISCARD requests are treated like WRITE requests. Otherwise we would +Any REQ_OP_DISCARD requests are treated like WRITE requests. Otherwise we would have all the DISCARD requests, and then the WRITE requests and then the FLUSH request. Consider the following example: diff --git a/Documentation/devicetree/bindings/ata/ahci-platform.txt b/Documentation/devicetree/bindings/ata/ahci-platform.txt index 5d5bd456d9d9..e30fd106df4f 100644 --- a/Documentation/devicetree/bindings/ata/ahci-platform.txt +++ b/Documentation/devicetree/bindings/ata/ahci-platform.txt @@ -10,6 +10,7 @@ PHYs. Required properties: - compatible : compatible string, one of: - "allwinner,sun4i-a10-ahci" + - "allwinner,sun8i-r40-ahci" - "brcm,iproc-ahci" - "hisilicon,hisi-ahci" - "cavium,octeon-7130-ahci" @@ -31,8 +32,10 @@ Optional properties: - clocks : a list of phandle + clock specifier pairs - resets : a list of phandle + reset specifier pairs - target-supply : regulator for SATA target power +- phy-supply : regulator for PHY power - phys : reference to the SATA PHY node - phy-names : must be "sata-phy" +- ahci-supply : regulator for AHCI controller - ports-implemented : Mask that indicates which ports that the HBA supports are available for software to use. Useful if PORTS_IMPL is not programmed by the BIOS, which is true with @@ -42,12 +45,13 @@ Required properties when using sub-nodes: - #address-cells : number of cells to encode an address - #size-cells : number of cells representing the size of an address +For allwinner,sun8i-r40-ahci, the reset propertie must be present. Sub-nodes required properties: - reg : the port number And at least one of the following properties: - phys : reference to the SATA PHY node -- target-supply : regulator for SATA target power +- target-supply : regulator for SATA target power Examples: sata@ffe08000 { diff --git a/Documentation/devicetree/bindings/ata/brcm,sata-brcm.txt b/Documentation/devicetree/bindings/ata/brcm,sata-brcm.txt index 0a5b3b47f217..7713a413c6a7 100644 --- a/Documentation/devicetree/bindings/ata/brcm,sata-brcm.txt +++ b/Documentation/devicetree/bindings/ata/brcm,sata-brcm.txt @@ -9,6 +9,7 @@ Required properties: "brcm,bcm7445-ahci" "brcm,bcm-nsp-ahci" "brcm,sata3-ahci" + "brcm,bcm63138-ahci" - reg : register mappings for AHCI and SATA_TOP_CTRL - reg-names : "ahci" and "top-ctrl" - interrupts : interrupt mapping for SATA IRQ diff --git a/Documentation/devicetree/bindings/hwmon/ina3221.txt b/Documentation/devicetree/bindings/hwmon/ina3221.txt new file mode 100644 index 000000000000..a7b25caa2b8e --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/ina3221.txt @@ -0,0 +1,44 @@ +Texas Instruments INA3221 Device Tree Bindings + +1) ina3221 node + Required properties: + - compatible: Must be "ti,ina3221" + - reg: I2C address + + Optional properties: + = The node contains optional child nodes for three channels = + = Each child node describes the information of input source = + + - #address-cells: Required only if a child node is present. Must be 1. + - #size-cells: Required only if a child node is present. Must be 0. + +2) child nodes + Required properties: + - reg: Must be 0, 1 or 2, corresponding to IN1, IN2 or IN3 port of INA3221 + + Optional properties: + - label: Name of the input source + - shunt-resistor-micro-ohms: Shunt resistor value in micro-Ohm + +Example: + +ina3221@40 { + compatible = "ti,ina3221"; + reg = <0x40>; + #address-cells = <1>; + #size-cells = <0>; + + input@0 { + reg = <0x0>; + status = "disabled"; + }; + input@1 { + reg = <0x1>; + shunt-resistor-micro-ohms = <5000>; + }; + input@2 { + reg = <0x2>; + label = "VDD_5V"; + shunt-resistor-micro-ohms = <5000>; + }; +}; diff --git a/Documentation/devicetree/bindings/hwmon/ltc2978.txt b/Documentation/devicetree/bindings/hwmon/ltc2978.txt index bf2a47bbdc58..b428a70a7cc0 100644 --- a/Documentation/devicetree/bindings/hwmon/ltc2978.txt +++ b/Documentation/devicetree/bindings/hwmon/ltc2978.txt @@ -15,6 +15,7 @@ Required properties: * "lltc,ltm2987" * "lltc,ltm4675" * "lltc,ltm4676" + * "lltc,ltm4686" - reg: I2C slave address Optional properties: @@ -30,6 +31,7 @@ Valid names of regulators depend on number of supplies supported per device: * ltc3880, ltc3882, ltc3886 : vout0 - vout1 * ltc3883 : vout0 * ltm4676 : vout0 - vout1 + * ltm4686 : vout0 - vout1 Example: ltc2978@5e { diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,geni-se.txt b/Documentation/devicetree/bindings/soc/qcom/qcom,geni-se.txt index ff92e5a41bed..dab7ca9f250c 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom,geni-se.txt +++ b/Documentation/devicetree/bindings/soc/qcom/qcom,geni-se.txt @@ -53,20 +53,8 @@ Required properties: - clocks: Serial engine core clock needed by the device. Qualcomm Technologies Inc. GENI Serial Engine based SPI Controller - -Required properties: -- compatible: Must contain "qcom,geni-spi". -- reg: Must contain SPI register location and length. -- interrupts: Must contain SPI controller interrupts. -- clock-names: Must contain "se". -- clocks: Serial engine core clock needed by the device. -- spi-max-frequency: Specifies maximum SPI clock frequency, units - Hz. -- #address-cells: Must be <1> to define a chip select address on - the SPI bus. -- #size-cells: Must be <0>. - -SPI slave nodes must be children of the SPI master node and conform to SPI bus -binding as described in Documentation/devicetree/bindings/spi/spi-bus.txt. +node binding is described in +Documentation/devicetree/bindings/spi/qcom,spi-geni-qcom.txt. Example: geniqup@8c0000 { @@ -103,17 +91,4 @@ Example: pinctrl-1 = <&qup_1_uart_3_sleep>; }; - spi0: spi@a84000 { - compatible = "qcom,geni-spi"; - reg = <0xa84000 0x4000>; - interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>; - clock-names = "se"; - clocks = <&clock_gcc GCC_QUPV3_WRAP0_S0_CLK>; - pinctrl-names = "default", "sleep"; - pinctrl-0 = <&qup_1_spi_2_active>; - pinctrl-1 = <&qup_1_spi_2_sleep>; - spi-max-frequency = <19200000>; - #address-cells = <1>; - #size-cells = <0>; - }; } diff --git a/Documentation/devicetree/bindings/spi/qcom,spi-geni-qcom.txt b/Documentation/devicetree/bindings/spi/qcom,spi-geni-qcom.txt new file mode 100644 index 000000000000..790311a42bf1 --- /dev/null +++ b/Documentation/devicetree/bindings/spi/qcom,spi-geni-qcom.txt @@ -0,0 +1,39 @@ +GENI based Qualcomm Universal Peripheral (QUP) Serial Peripheral Interface (SPI) + +The QUP v3 core is a GENI based AHB slave that provides a common data path +(an output FIFO and an input FIFO) for serial peripheral interface (SPI) +mini-core. + +SPI in master mode supports up to 50MHz, up to four chip selects, programmable +data path from 4 bits to 32 bits and numerous protocol variants. + +Required properties: +- compatible: Must contain "qcom,geni-spi". +- reg: Must contain SPI register location and length. +- interrupts: Must contain SPI controller interrupts. +- clock-names: Must contain "se". +- clocks: Serial engine core clock needed by the device. +- #address-cells: Must be <1> to define a chip select address on + the SPI bus. +- #size-cells: Must be <0>. + +SPI Controller nodes must be child of GENI based Qualcomm Universal +Peripharal. Please refer GENI based QUP wrapper controller node bindings +described in Documentation/devicetree/bindings/soc/qcom/qcom,geni-se.txt. + +SPI slave nodes must be children of the SPI master node and conform to SPI bus +binding as described in Documentation/devicetree/bindings/spi/spi-bus.txt. + +Example: + spi0: spi@a84000 { + compatible = "qcom,geni-spi"; + reg = <0xa84000 0x4000>; + interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>; + clock-names = "se"; + clocks = <&clock_gcc GCC_QUPV3_WRAP0_S0_CLK>; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&qup_1_spi_2_active>; + pinctrl-1 = <&qup_1_spi_2_sleep>; + #address-cells = <1>; + #size-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/spi/qcom,spi-qcom-qspi.txt b/Documentation/devicetree/bindings/spi/qcom,spi-qcom-qspi.txt new file mode 100644 index 000000000000..1d64b61f5171 --- /dev/null +++ b/Documentation/devicetree/bindings/spi/qcom,spi-qcom-qspi.txt @@ -0,0 +1,36 @@ +Qualcomm Quad Serial Peripheral Interface (QSPI) + +The QSPI controller allows SPI protocol communication in single, dual, or quad +wire transmission modes for read/write access to slaves such as NOR flash. + +Required properties: +- compatible: An SoC specific identifier followed by "qcom,qspi-v1", such as + "qcom,sdm845-qspi", "qcom,qspi-v1" +- reg: Should contain the base register location and length. +- interrupts: Interrupt number used by the controller. +- clocks: Should contain the core and AHB clock. +- clock-names: Should be "core" for core clock and "iface" for AHB clock. + +SPI slave nodes must be children of the SPI master node and can contain +properties described in Documentation/devicetree/bindings/spi/spi-bus.txt + +Example: + + qspi: spi@88df000 { + compatible = "qcom,sdm845-qspi", "qcom,qspi-v1"; + reg = <0x88df000 0x600>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>; + clock-names = "iface", "core"; + clocks = <&gcc GCC_QSPI_CNOC_PERIPH_AHB_CLK>, + <&gcc GCC_QSPI_CORE_CLK>; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <25000000>; + spi-tx-bus-width = <2>; + spi-rx-bus-width = <2>; + }; + }; diff --git a/Documentation/devicetree/bindings/spi/sh-msiof.txt b/Documentation/devicetree/bindings/spi/sh-msiof.txt index bfbc2035fb6b..4b836ad17b19 100644 --- a/Documentation/devicetree/bindings/spi/sh-msiof.txt +++ b/Documentation/devicetree/bindings/spi/sh-msiof.txt @@ -2,7 +2,9 @@ Renesas MSIOF spi controller Required properties: - compatible : "renesas,msiof-r8a7743" (RZ/G1M) + "renesas,msiof-r8a7744" (RZ/G1N) "renesas,msiof-r8a7745" (RZ/G1E) + "renesas,msiof-r8a774a1" (RZ/G2M) "renesas,msiof-r8a7790" (R-Car H2) "renesas,msiof-r8a7791" (R-Car M2-W) "renesas,msiof-r8a7792" (R-Car V2H) @@ -11,10 +13,14 @@ Required properties: "renesas,msiof-r8a7795" (R-Car H3) "renesas,msiof-r8a7796" (R-Car M3-W) "renesas,msiof-r8a77965" (R-Car M3-N) + "renesas,msiof-r8a77970" (R-Car V3M) + "renesas,msiof-r8a77980" (R-Car V3H) + "renesas,msiof-r8a77990" (R-Car E3) + "renesas,msiof-r8a77995" (R-Car D3) "renesas,msiof-sh73a0" (SH-Mobile AG5) "renesas,sh-mobile-msiof" (generic SH-Mobile compatibile device) "renesas,rcar-gen2-msiof" (generic R-Car Gen2 and RZ/G1 compatible device) - "renesas,rcar-gen3-msiof" (generic R-Car Gen3 compatible device) + "renesas,rcar-gen3-msiof" (generic R-Car Gen3 and RZ/G2 compatible device) "renesas,sh-msiof" (deprecated) When compatible with the generic version, nodes diff --git a/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.txt b/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.txt index 642d3fb1ef85..2864bc6b659c 100644 --- a/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.txt +++ b/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.txt @@ -2,7 +2,7 @@ Synopsys DesignWare AMBA 2.0 Synchronous Serial Interface. Required properties: - compatible : "snps,dw-apb-ssi" or "mscc,<soc>-spi", where soc is "ocelot" or - "jaguar2" + "jaguar2", or "amazon,alpine-dw-apb-ssi" - reg : The register base for the controller. For "mscc,<soc>-spi", a second register set is required (named ICPU_CFG:SPI_MST) - interrupts : One interrupt, used by the controller. diff --git a/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.txt b/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.txt index 4af132606b37..8d178a4503cf 100644 --- a/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.txt +++ b/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.txt @@ -3,6 +3,7 @@ Required properties: - compatible : - "fsl,imx7ulp-spi" for LPSPI compatible with the one integrated on i.MX7ULP soc + - "fsl,imx8qxp-spi" for LPSPI compatible with the one integrated on i.MX8QXP soc - reg : address and length of the lpspi master registers - interrupts : lpspi interrupt - clocks : lpspi clock specifier diff --git a/Documentation/devicetree/bindings/spi/spi-pxa2xx.txt b/Documentation/devicetree/bindings/spi/spi-pxa2xx.txt new file mode 100644 index 000000000000..0335a9bd2e8a --- /dev/null +++ b/Documentation/devicetree/bindings/spi/spi-pxa2xx.txt @@ -0,0 +1,24 @@ +PXA2xx SSP SPI Controller + +Required properties: +- compatible: Must be "marvell,mmp2-ssp". +- reg: Offset and length of the device's register set. +- interrupts: Should be the interrupt number. +- clocks: Should contain a single entry describing the clock input. +- #address-cells: Number of cells required to define a chip select address. +- #size-cells: Should be zero. + +Optional properties: +- cs-gpios: list of GPIO chip selects. See the SPI bus bindings, + Documentation/devicetree/bindings/spi/spi-bus.txt + +Child nodes represent devices on the SPI bus + See ../spi/spi-bus.txt + +Example: + ssp1: spi@d4035000 { + compatible = "marvell,mmp2-ssp"; + reg = <0xd4035000 0x1000>; + clocks = <&soc_clocks MMP2_CLK_SSP0>; + interrupts = <0>; + }; diff --git a/Documentation/devicetree/bindings/spi/spi-rspi.txt b/Documentation/devicetree/bindings/spi/spi-rspi.txt index 96fd58548f69..fc97ad64fbf2 100644 --- a/Documentation/devicetree/bindings/spi/spi-rspi.txt +++ b/Documentation/devicetree/bindings/spi/spi-rspi.txt @@ -3,7 +3,7 @@ Device tree configuration for Renesas RSPI/QSPI driver Required properties: - compatible : For Renesas Serial Peripheral Interface on legacy SH: "renesas,rspi-<soctype>", "renesas,rspi" as fallback. - For Renesas Serial Peripheral Interface on RZ/A1H: + For Renesas Serial Peripheral Interface on RZ/A: "renesas,rspi-<soctype>", "renesas,rspi-rz" as fallback. For Quad Serial Peripheral Interface on R-Car Gen2 and RZ/G1 devices: @@ -11,7 +11,9 @@ Required properties: Examples with soctypes are: - "renesas,rspi-sh7757" (SH) - "renesas,rspi-r7s72100" (RZ/A1H) + - "renesas,rspi-r7s9210" (RZ/A2) - "renesas,qspi-r8a7743" (RZ/G1M) + - "renesas,qspi-r8a7744" (RZ/G1N) - "renesas,qspi-r8a7745" (RZ/G1E) - "renesas,qspi-r8a7790" (R-Car H2) - "renesas,qspi-r8a7791" (R-Car M2-W) diff --git a/Documentation/devicetree/bindings/spi/spi-slave-mt27xx.txt b/Documentation/devicetree/bindings/spi/spi-slave-mt27xx.txt new file mode 100644 index 000000000000..c37e5a179b21 --- /dev/null +++ b/Documentation/devicetree/bindings/spi/spi-slave-mt27xx.txt @@ -0,0 +1,32 @@ +Binding for MTK SPI Slave controller + +Required properties: +- compatible: should be one of the following. + - mediatek,mt2712-spi-slave: for mt2712 platforms +- reg: Address and length of the register set for the device. +- interrupts: Should contain spi interrupt. +- clocks: phandles to input clocks. + It's clock gate, and should be <&infracfg CLK_INFRA_AO_SPI1>. +- clock-names: should be "spi" for the clock gate. + +Optional properties: +- assigned-clocks: it's mux clock, should be <&topckgen CLK_TOP_SPISLV_SEL>. +- assigned-clock-parents: parent of mux clock. + It's PLL, and should be one of the following. + - <&topckgen CLK_TOP_UNIVPLL1_D2>: specify parent clock 312MHZ. + It's the default one. + - <&topckgen CLK_TOP_UNIVPLL1_D4>: specify parent clock 156MHZ. + - <&topckgen CLK_TOP_UNIVPLL2_D4>: specify parent clock 104MHZ. + - <&topckgen CLK_TOP_UNIVPLL1_D8>: specify parent clock 78MHZ. + +Example: +- SoC Specific Portion: +spis1: spi@10013000 { + compatible = "mediatek,mt2712-spi-slave"; + reg = <0 0x10013000 0 0x100>; + interrupts = <GIC_SPI 283 IRQ_TYPE_LEVEL_LOW>; + clocks = <&infracfg CLK_INFRA_AO_SPI1>; + clock-names = "spi"; + assigned-clocks = <&topckgen CLK_TOP_SPISLV_SEL>; + assigned-clock-parents = <&topckgen CLK_TOP_UNIVPLL1_D2>; +}; diff --git a/Documentation/devicetree/bindings/spi/spi-sprd.txt b/Documentation/devicetree/bindings/spi/spi-sprd.txt new file mode 100644 index 000000000000..bad211a19da4 --- /dev/null +++ b/Documentation/devicetree/bindings/spi/spi-sprd.txt @@ -0,0 +1,26 @@ +Spreadtrum SPI Controller + +Required properties: +- compatible: Should be "sprd,sc9860-spi". +- reg: Offset and length of SPI controller register space. +- interrupts: Should contain SPI interrupt. +- clock-names: Should contain following entries: + "spi" for SPI clock, + "source" for SPI source (parent) clock, + "enable" for SPI module enable clock. +- clocks: List of clock input name strings sorted in the same order + as the clock-names property. +- #address-cells: The number of cells required to define a chip select + address on the SPI bus. Should be set to 1. +- #size-cells: Should be set to 0. + +Example: +spi0: spi@70a00000{ + compatible = "sprd,sc9860-spi"; + reg = <0 0x70a00000 0 0x1000>; + interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>; + clock-names = "spi", "source","enable"; + clocks = <&clk_spi0>, <&ext_26m>, <&clk_ap_apb_gates 5>; + #address-cells = <1>; + #size-cells = <0>; +}; diff --git a/Documentation/devicetree/bindings/spi/spi-stm32-qspi.txt b/Documentation/devicetree/bindings/spi/spi-stm32-qspi.txt new file mode 100644 index 000000000000..adeeb63e84b9 --- /dev/null +++ b/Documentation/devicetree/bindings/spi/spi-stm32-qspi.txt @@ -0,0 +1,44 @@ +* STMicroelectronics Quad Serial Peripheral Interface(QSPI) + +Required properties: +- compatible: should be "st,stm32f469-qspi" +- reg: the first contains the register location and length. + the second contains the memory mapping address and length +- reg-names: should contain the reg names "qspi" "qspi_mm" +- interrupts: should contain the interrupt for the device +- clocks: the phandle of the clock needed by the QSPI controller +- A pinctrl must be defined to set pins in mode of operation for QSPI transfer + +Optional properties: +- resets: must contain the phandle to the reset controller. + +A spi flash (NOR/NAND) must be a child of spi node and could have some +properties. Also see jedec,spi-nor.txt. + +Required properties: +- reg: chip-Select number (QSPI controller may connect 2 flashes) +- spi-max-frequency: max frequency of spi bus + +Optional property: +- spi-rx-bus-width: see ./spi-bus.txt for the description + +Example: + +qspi: spi@a0001000 { + compatible = "st,stm32f469-qspi"; + reg = <0xa0001000 0x1000>, <0x90000000 0x10000000>; + reg-names = "qspi", "qspi_mm"; + interrupts = <91>; + resets = <&rcc STM32F4_AHB3_RESET(QSPI)>; + clocks = <&rcc 0 STM32F4_AHB3_CLOCK(QSPI)>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_qspi0>; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-rx-bus-width = <4>; + spi-max-frequency = <108000000>; + ... + }; +}; diff --git a/Documentation/driver-api/mtdnand.rst b/Documentation/driver-api/mtdnand.rst index c55a6034c397..55447659b81f 100644 --- a/Documentation/driver-api/mtdnand.rst +++ b/Documentation/driver-api/mtdnand.rst @@ -180,10 +180,10 @@ by a chip select decoder. { struct nand_chip *this = mtd_to_nand(mtd); switch(cmd){ - case NAND_CTL_SETCLE: this->IO_ADDR_W |= CLE_ADRR_BIT; break; - case NAND_CTL_CLRCLE: this->IO_ADDR_W &= ~CLE_ADRR_BIT; break; - case NAND_CTL_SETALE: this->IO_ADDR_W |= ALE_ADRR_BIT; break; - case NAND_CTL_CLRALE: this->IO_ADDR_W &= ~ALE_ADRR_BIT; break; + case NAND_CTL_SETCLE: this->legacy.IO_ADDR_W |= CLE_ADRR_BIT; break; + case NAND_CTL_CLRCLE: this->legacy.IO_ADDR_W &= ~CLE_ADRR_BIT; break; + case NAND_CTL_SETALE: this->legacy.IO_ADDR_W |= ALE_ADRR_BIT; break; + case NAND_CTL_CLRALE: this->legacy.IO_ADDR_W &= ~ALE_ADRR_BIT; break; } } @@ -197,7 +197,7 @@ to read back the state of the pin. The function has no arguments and should return 0, if the device is busy (R/B pin is low) and 1, if the device is ready (R/B pin is high). If the hardware interface does not give access to the ready busy pin, then the function must not be defined -and the function pointer this->dev_ready is set to NULL. +and the function pointer this->legacy.dev_ready is set to NULL. Init function ------------- @@ -235,18 +235,18 @@ necessary information about the device. } /* Set address of NAND IO lines */ - this->IO_ADDR_R = baseaddr; - this->IO_ADDR_W = baseaddr; + this->legacy.IO_ADDR_R = baseaddr; + this->legacy.IO_ADDR_W = baseaddr; /* Reference hardware control function */ this->hwcontrol = board_hwcontrol; /* Set command delay time, see datasheet for correct value */ - this->chip_delay = CHIP_DEPENDEND_COMMAND_DELAY; + this->legacy.chip_delay = CHIP_DEPENDEND_COMMAND_DELAY; /* Assign the device ready function, if available */ - this->dev_ready = board_dev_ready; + this->legacy.dev_ready = board_dev_ready; this->eccmode = NAND_ECC_SOFT; /* Scan to find existence of the device */ - if (nand_scan (board_mtd, 1)) { + if (nand_scan (this, 1)) { err = -ENXIO; goto out_ior; } @@ -277,7 +277,7 @@ unregisters the partitions in the MTD layer. static void __exit board_cleanup (void) { /* Release resources, unregister device */ - nand_release (board_mtd); + nand_release (mtd_to_nand(board_mtd)); /* unmap physical address */ iounmap(baseaddr); @@ -336,17 +336,17 @@ connected to an address decoder. struct nand_chip *this = mtd_to_nand(mtd); /* Deselect all chips */ - this->IO_ADDR_R &= ~BOARD_NAND_ADDR_MASK; - this->IO_ADDR_W &= ~BOARD_NAND_ADDR_MASK; + this->legacy.IO_ADDR_R &= ~BOARD_NAND_ADDR_MASK; + this->legacy.IO_ADDR_W &= ~BOARD_NAND_ADDR_MASK; switch (chip) { case 0: - this->IO_ADDR_R |= BOARD_NAND_ADDR_CHIP0; - this->IO_ADDR_W |= BOARD_NAND_ADDR_CHIP0; + this->legacy.IO_ADDR_R |= BOARD_NAND_ADDR_CHIP0; + this->legacy.IO_ADDR_W |= BOARD_NAND_ADDR_CHIP0; break; .... case n: - this->IO_ADDR_R |= BOARD_NAND_ADDR_CHIPn; - this->IO_ADDR_W |= BOARD_NAND_ADDR_CHIPn; + this->legacy.IO_ADDR_R |= BOARD_NAND_ADDR_CHIPn; + this->legacy.IO_ADDR_W |= BOARD_NAND_ADDR_CHIPn; break; } } diff --git a/Documentation/hwmon/ina3221 b/Documentation/hwmon/ina3221 index 0ff74854cb2e..4b82cbfb551c 100644 --- a/Documentation/hwmon/ina3221 +++ b/Documentation/hwmon/ina3221 @@ -21,6 +21,8 @@ and power are calculated host-side from these. Sysfs entries ------------- +in[123]_label Voltage channel labels +in[123]_enable Voltage channel enable controls in[123]_input Bus voltage(mV) channels curr[123]_input Current(mA) measurement channels shunt[123]_resistor Shunt resistance(uOhm) channels diff --git a/Documentation/hwmon/lm75 b/Documentation/hwmon/lm75 index ac95edfcd907..2f1120f88c16 100644 --- a/Documentation/hwmon/lm75 +++ b/Documentation/hwmon/lm75 @@ -17,8 +17,8 @@ Supported chips: Addresses scanned: none Datasheet: Publicly available at the Maxim website http://www.maximintegrated.com/ - * Maxim MAX6625, MAX6626 - Prefixes: 'max6625', 'max6626' + * Maxim MAX6625, MAX6626, MAX31725, MAX31726 + Prefixes: 'max6625', 'max6626', 'max31725', 'max31726' Addresses scanned: none Datasheet: Publicly available at the Maxim website http://www.maxim-ic.com/ @@ -86,7 +86,7 @@ The LM75 is essentially an industry standard; there may be other LM75 clones not listed here, with or without various enhancements, that are supported. The clones are not detected by the driver, unless they reproduce the exact register tricks of the original LM75, and must -therefore be instantiated explicitly. Higher resolution up to 12-bit +therefore be instantiated explicitly. Higher resolution up to 16-bit is supported by this driver, other specific enhancements are not. The LM77 is not supported, contrary to what we pretended for a long time. diff --git a/Documentation/hwmon/ltc2978 b/Documentation/hwmon/ltc2978 index 9a49d3c90cd1..dfb2caa401d9 100644 --- a/Documentation/hwmon/ltc2978 +++ b/Documentation/hwmon/ltc2978 @@ -55,6 +55,10 @@ Supported chips: Prefix: 'ltm4676' Addresses scanned: - Datasheet: http://www.linear.com/product/ltm4676 + * Analog Devices LTM4686 + Prefix: 'ltm4686' + Addresses scanned: - + Datasheet: http://www.analog.com/ltm4686 Author: Guenter Roeck <linux@roeck-us.net> @@ -76,6 +80,7 @@ additional components on a single die. The chip is instantiated and reported as two separate chips on two different I2C bus addresses. LTM4675 is a dual 9A or single 18A μModule regulator LTM4676 is a dual 13A or single 26A uModule regulator. +LTM4686 is a dual 10A or single 20A uModule regulator. Usage Notes diff --git a/Documentation/hwmon/mc13783-adc b/Documentation/hwmon/mc13783-adc index d0e7b3fa9e75..05ccc9f159f1 100644 --- a/Documentation/hwmon/mc13783-adc +++ b/Documentation/hwmon/mc13783-adc @@ -2,12 +2,12 @@ Kernel driver mc13783-adc ========================= Supported chips: - * Freescale Atlas MC13783 + * Freescale MC13783 Prefix: 'mc13783' - Datasheet: http://www.freescale.com/files/rf_if/doc/data_sheet/MC13783.pdf?fsrch=1 - * Freescale Atlas MC13892 + Datasheet: https://www.nxp.com/docs/en/data-sheet/MC13783.pdf + * Freescale MC13892 Prefix: 'mc13892' - Datasheet: http://cache.freescale.com/files/analog/doc/data_sheet/MC13892.pdf?fsrch=1&sr=1 + Datasheet: https://www.nxp.com/docs/en/data-sheet/MC13892.pdf Authors: Sascha Hauer <s.hauer@pengutronix.de> diff --git a/Documentation/mtd/nand/pxa3xx-nand.txt b/Documentation/mtd/nand/pxa3xx-nand.txt deleted file mode 100644 index 1074cbc67ec6..000000000000 --- a/Documentation/mtd/nand/pxa3xx-nand.txt +++ /dev/null @@ -1,113 +0,0 @@ - -About this document -=================== - -Some notes about Marvell's NAND controller available in PXA and Armada 370/XP -SoC (aka NFCv1 and NFCv2), with an emphasis on the latter. - -NFCv2 controller background -=========================== - -The controller has a 2176 bytes FIFO buffer. Therefore, in order to support -larger pages, I/O operations on 4 KiB and 8 KiB pages is done with a set of -chunked transfers. - -For instance, if we choose a 2048 data chunk and set "BCH" ECC (see below) -we'll have this layout in the pages: - - ------------------------------------------------------------------------------ - | 2048B data | 32B spare | 30B ECC || 2048B data | 32B spare | 30B ECC | ... | - ------------------------------------------------------------------------------ - -The driver reads the data and spare portions independently and builds an internal -buffer with this layout (in the 4 KiB page case): - - ------------------------------------------ - | 4096B data | 64B spare | - ------------------------------------------ - -Also, for the READOOB command the driver disables the ECC and reads a 'spare + ECC' -OOB, one per chunk read. - - ------------------------------------------------------------------- - | 4096B data | 32B spare | 30B ECC | 32B spare | 30B ECC | - ------------------------------------------------------------------- - -So, in order to achieve reading (for instance), we issue several READ0 commands -(with some additional controller-specific magic) and read two chunks of 2080B -(2048 data + 32 spare) each. -The driver accommodates this data to expose the NAND core a contiguous buffer -(4096 data + spare) or (4096 + spare + ECC + spare + ECC). - -ECC -=== - -The controller has built-in hardware ECC capabilities. In addition it is -configurable between two modes: 1) Hamming, 2) BCH. - -Note that the actual BCH mode: BCH-4 or BCH-8 will depend on the way -the controller is configured to transfer the data. - -In the BCH mode the ECC code will be calculated for each transferred chunk -and expected to be located (when reading/programming) right after the spare -bytes as the figure above shows. - -So, repeating the above scheme, a 2048B data chunk will be followed by 32B -spare, and then the ECC controller will read/write the ECC code (30B in -this case): - - ------------------------------------ - | 2048B data | 32B spare | 30B ECC | - ------------------------------------ - -If the ECC mode is 'BCH' then the ECC is *always* 30 bytes long. -If the ECC mode is 'Hamming' the ECC is 6 bytes long, for each 512B block. -So in Hamming mode, a 2048B page will have a 24B ECC. - -Despite all of the above, the controller requires the driver to only read or -write in multiples of 8-bytes, because the data buffer is 64-bits. - -OOB -=== - -Because of the above scheme, and because the "spare" OOB is really located in -the middle of a page, spare OOB cannot be read or write independently of the -data area. In other words, in order to read the OOB (aka READOOB), the entire -page (aka READ0) has to be read. - -In the same sense, in order to write to the spare OOB the driver has to write -an *entire* page. - -Factory bad blocks handling -=========================== - -Given the ECC BCH requires to layout the device's pages in a split -data/OOB/data/OOB way, the controller has a view of the flash page that's -different from the specified (aka the manufacturer's) view. In other words, - -Factory view: - - ----------------------------------------------- - | Data |x OOB | - ----------------------------------------------- - -Driver's view: - - ----------------------------------------------- - | Data | OOB | Data x | OOB | - ----------------------------------------------- - -It can be seen from the above, that the factory bad block marker must be -searched within the 'data' region, and not in the usual OOB region. - -In addition, this means under regular usage the driver will write such -position (since it belongs to the data region) and every used block is -likely to be marked as bad. - -For this reason, marking the block as bad in the OOB is explicitly -disabled by using the NAND_BBT_NO_OOB_BBM option in the driver. The rationale -for this is that there's no point in marking a block as bad, because good -blocks are also 'marked as bad' (in the OOB BBM sense) under normal usage. - -Instead, the driver relies on the bad block table alone, and should only perform -the bad block scan on the very first time (when the device hasn't been used). diff --git a/Documentation/process/code-of-conduct-interpretation.rst b/Documentation/process/code-of-conduct-interpretation.rst new file mode 100644 index 000000000000..e899f14a4ba2 --- /dev/null +++ b/Documentation/process/code-of-conduct-interpretation.rst @@ -0,0 +1,156 @@ +.. _code_of_conduct_interpretation: + +Linux Kernel Contributor Covenant Code of Conduct Interpretation +================================================================ + +The :ref:`code_of_conduct` is a general document meant to +provide a set of rules for almost any open source community. Every +open-source community is unique and the Linux kernel is no exception. +Because of this, this document describes how we in the Linux kernel +community will interpret it. We also do not expect this interpretation +to be static over time, and will adjust it as needed. + +The Linux kernel development effort is a very personal process compared +to "traditional" ways of developing software. Your contributions and +ideas behind them will be carefully reviewed, often resulting in +critique and criticism. The review will almost always require +improvements before the material can be included in the +kernel. Know that this happens because everyone involved wants to see +the best possible solution for the overall success of Linux. This +development process has been proven to create the most robust operating +system kernel ever, and we do not want to do anything to cause the +quality of submission and eventual result to ever decrease. + +Maintainers +----------- + +The Code of Conduct uses the term "maintainers" numerous times. In the +kernel community, a "maintainer" is anyone who is responsible for a +subsystem, driver, or file, and is listed in the MAINTAINERS file in the +kernel source tree. + +Responsibilities +---------------- + +The Code of Conduct mentions rights and responsibilities for +maintainers, and this needs some further clarifications. + +First and foremost, it is a reasonable expectation to have maintainers +lead by example. + +That being said, our community is vast and broad, and there is no new +requirement for maintainers to unilaterally handle how other people +behave in the parts of the community where they are active. That +responsibility is upon all of us, and ultimately the Code of Conduct +documents final escalation paths in case of unresolved concerns +regarding conduct issues. + +Maintainers should be willing to help when problems occur, and work with +others in the community when needed. Do not be afraid to reach out to +the Technical Advisory Board (TAB) or other maintainers if you're +uncertain how to handle situations that come up. It will not be +considered a violation report unless you want it to be. If you are +uncertain about approaching the TAB or any other maintainers, please +reach out to our conflict mediator, Mishi Choudhary <mishi@linux.com>. + +In the end, "be kind to each other" is really what the end goal is for +everybody. We know everyone is human and we all fail at times, but the +primary goal for all of us should be to work toward amicable resolutions +of problems. Enforcement of the code of conduct will only be a last +resort option. + +Our goal of creating a robust and technically advanced operating system +and the technical complexity involved naturally require expertise and +decision-making. + +The required expertise varies depending on the area of contribution. It +is determined mainly by context and technical complexity and only +secondary by the expectations of contributors and maintainers. + +Both the expertise expectations and decision-making are subject to +discussion, but at the very end there is a basic necessity to be able to +make decisions in order to make progress. This prerogative is in the +hands of maintainers and project's leadership and is expected to be used +in good faith. + +As a consequence, setting expertise expectations, making decisions and +rejecting unsuitable contributions are not viewed as a violation of the +Code of Conduct. + +While maintainers are in general welcoming to newcomers, their capacity +of helping contributors overcome the entry hurdles is limited, so they +have to set priorities. This, also, is not to be seen as a violation of +the Code of Conduct. The kernel community is aware of that and provides +entry level programs in various forms like kernelnewbies.org. + +Scope +----- + +The Linux kernel community primarily interacts on a set of public email +lists distributed around a number of different servers controlled by a +number of different companies or individuals. All of these lists are +defined in the MAINTAINERS file in the kernel source tree. Any emails +sent to those mailing lists are considered covered by the Code of +Conduct. + +Developers who use the kernel.org bugzilla, and other subsystem bugzilla +or bug tracking tools should follow the guidelines of the Code of +Conduct. The Linux kernel community does not have an "official" project +email address, or "official" social media address. Any activity +performed using a kernel.org email account must follow the Code of +Conduct as published for kernel.org, just as any individual using a +corporate email account must follow the specific rules of that +corporation. + +The Code of Conduct does not prohibit continuing to include names, email +addresses, and associated comments in mailing list messages, kernel +change log messages, or code comments. + +Interaction in other forums is covered by whatever rules apply to said +forums and is in general not covered by the Code of Conduct. Exceptions +may be considered for extreme circumstances. + +Contributions submitted for the kernel should use appropriate language. +Content that already exists predating the Code of Conduct will not be +addressed now as a violation. Inappropriate language can be seen as a +bug, though; such bugs will be fixed more quickly if any interested +parties submit patches to that effect. Expressions that are currently +part of the user/kernel API, or reflect terminology used in published +standards or specifications, are not considered bugs. + +Enforcement +----------- + +The address listed in the Code of Conduct goes to the Code of Conduct +Committee. The exact members receiving these emails at any given time +are listed at https://kernel.org/code-of-conduct.html. Members can not +access reports made before they joined or after they have left the +committee. + +The initial Code of Conduct Committee consists of volunteer members of +the TAB, as well as a professional mediator acting as a neutral third +party. The first task of the committee is to establish documented +processes, which will be made public. + +Any member of the committee, including the mediator, can be contacted +directly if a reporter does not wish to include the full committee in a +complaint or concern. + +The Code of Conduct Committee reviews the cases according to the +processes (see above) and consults with the TAB as needed and +appropriate, for instance to request and receive information about the +kernel community. + +Any decisions by the committee will be brought to the TAB, for +implementation of enforcement with the relevant maintainers if needed. +A decision by the Code of Conduct Committee can be overturned by the TAB +by a two-thirds vote. + +At quarterly intervals, the Code of Conduct Committee and TAB will +provide a report summarizing the anonymised reports that the Code of +Conduct committee has received and their status, as well details of any +overridden decisions including complete and identifiable voting details. + +We expect to establish a different process for Code of Conduct Committee +staffing beyond the bootstrap period. This document will be updated +with that information when this occurs. diff --git a/Documentation/process/code-of-conduct.rst b/Documentation/process/code-of-conduct.rst index ab7c24b5478c..be50294aebd5 100644 --- a/Documentation/process/code-of-conduct.rst +++ b/Documentation/process/code-of-conduct.rst @@ -1,3 +1,5 @@ +.. _code_of_conduct: + Contributor Covenant Code of Conduct ++++++++++++++++++++++++++++++++++++ @@ -63,19 +65,22 @@ Enforcement =========== Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the Technical Advisory Board (TAB) at -<tab@lists.linux-foundation.org>. All complaints will be reviewed and -investigated and will result in a response that is deemed necessary and -appropriate to the circumstances. The TAB is obligated to maintain -confidentiality with regard to the reporter of an incident. Further details of -specific enforcement policies may be posted separately. - -Maintainers who do not follow or enforce the Code of Conduct in good faith may -face temporary or permanent repercussions as determined by other members of the -project’s leadership. +reported by contacting the Code of Conduct Committee at +<conduct@kernel.org>. All complaints will be reviewed and investigated +and will result in a response that is deemed necessary and appropriate +to the circumstances. The Code of Conduct Committee is obligated to +maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted +separately. Attribution =========== This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +Interpretation +============== + +See the :ref:`code_of_conduct_interpretation` document for how the Linux +kernel community will be interpreting this document. diff --git a/Documentation/process/index.rst b/Documentation/process/index.rst index 9ae3e317bddf..42691e2880eb 100644 --- a/Documentation/process/index.rst +++ b/Documentation/process/index.rst @@ -21,6 +21,7 @@ Below are the essential guides that every developer should read. howto code-of-conduct + code-of-conduct-interpretation development-process submitting-patches coding-style diff --git a/LICENSES/other/CC-BY-SA-4.0 b/LICENSES/other/CC-BY-SA-4.0 deleted file mode 100644 index f9158e831e79..000000000000 --- a/LICENSES/other/CC-BY-SA-4.0 +++ /dev/null @@ -1,397 +0,0 @@ -Valid-License-Identifier: CC-BY-SA-4.0 -SPDX-URL: https://spdx.org/licenses/CC-BY-SA-4.0 -Usage-Guide: - To use the Creative Commons Attribution Share Alike 4.0 International - license put the following SPDX tag/value pair into a comment according to - the placement guidelines in the licensing rules documentation: - SPDX-License-Identifier: CC-BY-SA-4.0 -License-Text: - -Creative Commons Attribution-ShareAlike 4.0 International - -Creative Commons Corporation ("Creative Commons") is not a law firm and -does not provide legal services or legal advice. Distribution of Creative -Commons public licenses does not create a lawyer-client or other -relationship. Creative Commons makes its licenses and related information -available on an "as-is" basis. Creative Commons gives no warranties -regarding its licenses, any material licensed under their terms and -conditions, or any related information. Creative Commons disclaims all -liability for damages resulting from their use to the fullest extent -possible. - -Using Creative Commons Public Licenses - -Creative Commons public licenses provide a standard set of terms and -conditions that creators and other rights holders may use to share original -works of authorship and other material subject to copyright and certain -other rights specified in the public license below. The following -considerations are for informational purposes only, are not exhaustive, and -do not form part of our licenses. - -Considerations for licensors: Our public licenses are intended for use by -those authorized to give the public permission to use material in ways -otherwise restricted by copyright and certain other rights. Our licenses -are irrevocable. Licensors should read and understand the terms and -conditions of the license they choose before applying it. Licensors should -also secure all rights necessary before applying our licenses so that the -public can reuse the material as expected. Licensors should clearly mark -any material not subject to the license. This includes other CC-licensed -material, or material used under an exception or limitation to -copyright. More considerations for licensors : -wiki.creativecommons.org/Considerations_for_licensors - -Considerations for the public: By using one of our public licenses, a -licensor grants the public permission to use the licensed material under -specified terms and conditions. If the licensor's permission is not -necessary for any reason - for example, because of any applicable exception -or limitation to copyright - then that use is not regulated by the -license. Our licenses grant only permissions under copyright and certain -other rights that a licensor has authority to grant. Use of the licensed -material may still be restricted for other reasons, including because -others have copyright or other rights in the material. A licensor may make -special requests, such as asking that all changes be marked or described. - -Although not required by our licenses, you are encouraged to respect those -requests where reasonable. More considerations for the public : -wiki.creativecommons.org/Considerations_for_licensees - -Creative Commons Attribution-ShareAlike 4.0 International Public License - -By exercising the Licensed Rights (defined below), You accept and agree to -be bound by the terms and conditions of this Creative Commons -Attribution-ShareAlike 4.0 International Public License ("Public -License"). To the extent this Public License may be interpreted as a -contract, You are granted the Licensed Rights in consideration of Your -acceptance of these terms and conditions, and the Licensor grants You such -rights in consideration of benefits the Licensor receives from making the -Licensed Material available under these terms and conditions. - -Section 1 - Definitions. - - a. Adapted Material means material subject to Copyright and Similar - Rights that is derived from or based upon the Licensed Material and - in which the Licensed Material is translated, altered, arranged, - transformed, or otherwise modified in a manner requiring permission - under the Copyright and Similar Rights held by the Licensor. For - purposes of this Public License, where the Licensed Material is a - musical work, performance, or sound recording, Adapted Material is - always produced where the Licensed Material is synched in timed - relation with a moving image. - - b. Adapter's License means the license You apply to Your Copyright and - Similar Rights in Your contributions to Adapted Material in - accordance with the terms and conditions of this Public License. - - c. BY-SA Compatible License means a license listed at - creativecommons.org/compatiblelicenses, approved by Creative Commons - as essentially the equivalent of this Public License. - - d. Copyright and Similar Rights means copyright and/or similar rights - closely related to copyright including, without limitation, - performance, broadcast, sound recording, and Sui Generis Database - Rights, without regard to how the rights are labeled or - categorized. For purposes of this Public License, the rights - specified in Section 2(b)(1)-(2) are not Copyright and Similar - Rights. - - e. Effective Technological Measures means those measures that, in the - absence of proper authority, may not be circumvented under laws - fulfilling obligations under Article 11 of the WIPO Copyright Treaty - adopted on December 20, 1996, and/or similar international - agreements. - - f. Exceptions and Limitations means fair use, fair dealing, and/or any - other exception or limitation to Copyright and Similar Rights that - applies to Your use of the Licensed Material. - - g. License Elements means the license attributes listed in the name of - a Creative Commons Public License. The License Elements of this - Public License are Attribution and ShareAlike. - - h. Licensed Material means the artistic or literary work, database, or - other material to which the Licensor applied this Public License. - - i. Licensed Rights means the rights granted to You subject to the terms - and conditions of this Public License, which are limited to all - Copyright and Similar Rights that apply to Your use of the Licensed - Material and that the Licensor has authority to license. - - j. Licensor means the individual(s) or entity(ies) granting rights - under this Public License. - - k. Share means to provide material to the public by any means or - process that requires permission under the Licensed Rights, such as - reproduction, public display, public performance, distribution, - dissemination, communication, or importation, and to make material - available to the public including in ways that members of the public - may access the material from a place and at a time individually - chosen by them. - - l. Sui Generis Database Rights means rights other than copyright - resulting from Directive 96/9/EC of the European Parliament and of - the Council of 11 March 1996 on the legal protection of databases, - as amended and/or succeeded, as well as other essentially equivalent - rights anywhere in the world. m. You means the individual or entity - exercising the Licensed Rights under this Public License. Your has a - corresponding meaning. - -Section 2 - Scope. - - a. License grant. - - 1. Subject to the terms and conditions of this Public License, the - Licensor hereby grants You a worldwide, royalty-free, - non-sublicensable, non-exclusive, irrevocable license to - exercise the Licensed Rights in the Licensed Material to: - - A. reproduce and Share the Licensed Material, in whole or in part; and - - B. produce, reproduce, and Share Adapted Material. - - 2. Exceptions and Limitations. For the avoidance of doubt, where - Exceptions and Limitations apply to Your use, this Public - License does not apply, and You do not need to comply with its - terms and conditions. - - 3. Term. The term of this Public License is specified in Section 6(a). - - 4. Media and formats; technical modifications allowed. The Licensor - authorizes You to exercise the Licensed Rights in all media and - formats whether now known or hereafter created, and to make - technical modifications necessary to do so. The Licensor waives - and/or agrees not to assert any right or authority to forbid You - from making technical modifications necessary to exercise the - Licensed Rights, including technical modifications necessary to - circumvent Effective Technological Measures. For purposes of - this Public License, simply making modifications authorized by - this Section 2(a)(4) never produces Adapted Material. - - 5. Downstream recipients. - - A. Offer from the Licensor - Licensed Material. Every recipient - of the Licensed Material automatically receives an offer - from the Licensor to exercise the Licensed Rights under the - terms and conditions of this Public License. - - B. Additional offer from the Licensor - Adapted Material. Every - recipient of Adapted Material from You automatically - receives an offer from the Licensor to exercise the Licensed - Rights in the Adapted Material under the conditions of the - Adapter's License You apply. - - C. No downstream restrictions. You may not offer or impose any - additional or different terms or conditions on, or apply any - Effective Technological Measures to, the Licensed Material - if doing so restricts exercise of the Licensed Rights by any - recipient of the Licensed Material. - - 6. No endorsement. Nothing in this Public License constitutes or - may be construed as permission to assert or imply that You are, - or that Your use of the Licensed Material is, connected with, or - sponsored, endorsed, or granted official status by, the Licensor - or others designated to receive attribution as provided in - Section 3(a)(1)(A)(i). - - b. Other rights. - - 1. Moral rights, such as the right of integrity, are not licensed - under this Public License, nor are publicity, privacy, and/or - other similar personality rights; however, to the extent - possible, the Licensor waives and/or agrees not to assert any - such rights held by the Licensor to the limited extent necessary - to allow You to exercise the Licensed Rights, but not otherwise. - - 2. Patent and trademark rights are not licensed under this Public - License. - - 3. To the extent possible, the Licensor waives any right to collect - royalties from You for the exercise of the Licensed Rights, - whether directly or through a collecting society under any - voluntary or waivable statutory or compulsory licensing - scheme. In all other cases the Licensor expressly reserves any - right to collect such royalties. - -Section 3 - License Conditions. - -Your exercise of the Licensed Rights is expressly made subject to the -following conditions. - - a. Attribution. - - 1. If You Share the Licensed Material (including in modified form), - You must: - - A. retain the following if it is supplied by the Licensor with - the Licensed Material: - - i. identification of the creator(s) of the Licensed - Material and any others designated to receive - attribution, in any reasonable manner requested by the - Licensor (including by pseudonym if designated); - - ii. a copyright notice; - - iii. a notice that refers to this Public License; - - iv. a notice that refers to the disclaimer of warranties; - - v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable; - - B. indicate if You modified the Licensed Material and retain an - indication of any previous modifications; and - - C. indicate the Licensed Material is licensed under this Public - License, and include the text of, or the URI or hyperlink to, - this Public License. - - 2. You may satisfy the conditions in Section 3(a)(1) in any - reasonable manner based on the medium, means, and context in - which You Share the Licensed Material. For example, it may be - reasonable to satisfy the conditions by providing a URI or - hyperlink to a resource that includes the required information. - - 3. If requested by the Licensor, You must remove any of the - information required by Section 3(a)(1)(A) to the extent - reasonably practicable. b. ShareAlike.In addition to the - conditions in Section 3(a), if You Share Adapted Material You - produce, the following conditions also apply. - - 1. The Adapter's License You apply must be a Creative Commons - license with the same License Elements, this version or - later, or a BY-SA Compatible License. - - 2. You must include the text of, or the URI or hyperlink to, the - Adapter's License You apply. You may satisfy this condition - in any reasonable manner based on the medium, means, and - context in which You Share Adapted Material. - - 3. You may not offer or impose any additional or different terms - or conditions on, or apply any Effective Technological - Measures to, Adapted Material that restrict exercise of the - rights granted under the Adapter's License You apply. - -Section 4 - Sui Generis Database Rights. - -Where the Licensed Rights include Sui Generis Database Rights that apply to -Your use of the Licensed Material: - - a. for the avoidance of doubt, Section 2(a)(1) grants You the right to - extract, reuse, reproduce, and Share all or a substantial portion of - the contents of the database; - - b. if You include all or a substantial portion of the database contents - in a database in which You have Sui Generis Database Rights, then - the database in which You have Sui Generis Database Rights (but not - its individual contents) is Adapted Material, including for purposes - of Section 3(b); and - - c. You must comply with the conditions in Section 3(a) if You Share all - or a substantial portion of the contents of the database. - - For the avoidance of doubt, this Section 4 supplements and does not - replace Your obligations under this Public License where the Licensed - Rights include other Copyright and Similar Rights. - -Section 5 - Disclaimer of Warranties and Limitation of Liability. - - a. Unless otherwise separately undertaken by the Licensor, to the - extent possible, the Licensor offers the Licensed Material as-is and - as-available, and makes no representations or warranties of any kind - concerning the Licensed Material, whether express, implied, - statutory, or other. This includes, without limitation, warranties - of title, merchantability, fitness for a particular purpose, - non-infringement, absence of latent or other defects, accuracy, or - the presence or absence of errors, whether or not known or - discoverable. Where disclaimers of warranties are not allowed in - full or in part, this disclaimer may not apply to You. - - b. To the extent possible, in no event will the Licensor be liable to - You on any legal theory (including, without limitation, negligence) - or otherwise for any direct, special, indirect, incidental, - consequential, punitive, exemplary, or other losses, costs, - expenses, or damages arising out of this Public License or use of - the Licensed Material, even if the Licensor has been advised of the - possibility of such losses, costs, expenses, or damages. Where a - limitation of liability is not allowed in full or in part, this - limitation may not apply to You. - - c. The disclaimer of warranties and limitation of liability provided - above shall be interpreted in a manner that, to the extent possible, - most closely approximates an absolute disclaimer and waiver of all - liability. - -Section 6 - Term and Termination. - - a. This Public License applies for the term of the Copyright and - Similar Rights licensed here. However, if You fail to comply with - this Public License, then Your rights under this Public License - terminate automatically. - - b. Where Your right to use the Licensed Material has terminated under - Section 6(a), it reinstates: - - 1. automatically as of the date the violation is cured, provided it - is cured within 30 days of Your discovery of the violation; or - - 2. upon express reinstatement by the Licensor. - - c. For the avoidance of doubt, this Section 6(b) does not affect any - right the Licensor may have to seek remedies for Your violations of - this Public License. - - d. For the avoidance of doubt, the Licensor may also offer the Licensed - Material under separate terms or conditions or stop distributing the - Licensed Material at any time; however, doing so will not terminate - this Public License. - - e. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. - -Section 7 - Other Terms and Conditions. - - a. The Licensor shall not be bound by any additional or different terms - or conditions communicated by You unless expressly agreed. - - b. Any arrangements, understandings, or agreements regarding the - Licensed Material not stated herein are separate from and - independent of the terms and conditions of this Public License. - -Section 8 - Interpretation. - - a. For the avoidance of doubt, this Public License does not, and shall - not be interpreted to, reduce, limit, restrict, or impose conditions - on any use of the Licensed Material that could lawfully be made - without permission under this Public License. - - b. To the extent possible, if any provision of this Public License is - deemed unenforceable, it shall be automatically reformed to the - minimum extent necessary to make it enforceable. If the provision - cannot be reformed, it shall be severed from this Public License - without affecting the enforceability of the remaining terms and - conditions. - - c. No term or condition of this Public License will be waived and no - failure to comply consented to unless expressly agreed to by the - Licensor. - - d. Nothing in this Public License constitutes or may be interpreted as - a limitation upon, or waiver of, any privileges and immunities that - apply to the Licensor or You, including from the legal processes of - any jurisdiction or authority. - -Creative Commons is not a party to its public licenses. Notwithstanding, -Creative Commons may elect to apply one of its public licenses to material -it publishes and in those instances will be considered the "Licensor." The -text of the Creative Commons public licenses is dedicated to the public -domain under the CC0 Public Domain Dedication. Except for the limited -purpose of indicating that material is shared under a Creative Commons -public license or as otherwise permitted by the Creative Commons policies -published at creativecommons.org/policies, Creative Commons does not -authorize the use of the trademark "Creative Commons" or any other -trademark or logo of Creative Commons without its prior written consent -including, without limitation, in connection with any unauthorized -modifications to any of its public licenses or any other arrangements, -understandings, or agreements concerning use of licensed material. For the -avoidance of doubt, this paragraph does not form part of the public -licenses. - -Creative Commons may be contacted at creativecommons.org. diff --git a/MAINTAINERS b/MAINTAINERS index 6ac000cc006d..40f5ec061896 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3006,6 +3006,14 @@ S: Supported F: drivers/gpio/gpio-brcmstb.c F: Documentation/devicetree/bindings/gpio/brcm,brcmstb-gpio.txt +BROADCOM BRCMSTB I2C DRIVER +M: Kamal Dasu <kdasu.kdev@gmail.com> +L: linux-i2c@vger.kernel.org +L: bcm-kernel-feedback-list@broadcom.com +S: Supported +F: drivers/i2c/busses/i2c-brcmstb.c +F: Documentation/devicetree/bindings/i2c/i2c-brcmstb.txt + BROADCOM BRCMSTB USB2 and USB3 PHY DRIVER M: Al Cooper <alcooperx@gmail.com> L: linux-kernel@vger.kernel.org @@ -3113,6 +3121,15 @@ S: Maintained F: Documentation/devicetree/bindings/memory-controllers/brcm,dpfe-cpu.txt F: drivers/memory/brcmstb_dpfe.c +BROADCOM SPI DRIVER +M: Kamal Dasu <kdasu.kdev@gmail.com> +M: bcm-kernel-feedback-list@broadcom.com +S: Maintained +F: Documentation/devicetree/bindings/spi/brcm,spi-bcm-qspi.txt +F: drivers/spi/spi-bcm-qspi.* +F: drivers/spi/spi-brcmstb-qspi.c +F: drivers/spi/spi-iproc-qspi.c + BROADCOM SYSTEMPORT ETHERNET DRIVER M: Florian Fainelli <f.fainelli@gmail.com> L: netdev@vger.kernel.org @@ -3673,6 +3690,12 @@ S: Maintained F: Documentation/devicetree/bindings/media/coda.txt F: drivers/media/platform/coda/ +CODE OF CONDUCT +M: Greg Kroah-Hartman <gregkh@linuxfoundation.org> +S: Supported +F: Documentation/process/code-of-conduct.rst +F: Documentation/process/code-of-conduct-interpretation.rst + COMMON CLK FRAMEWORK M: Michael Turquette <mturquette@baylibre.com> M: Stephen Boyd <sboyd@kernel.org> @@ -6454,6 +6477,7 @@ F: Documentation/devicetree/bindings/hwmon/ F: Documentation/hwmon/ F: drivers/hwmon/ F: include/linux/hwmon*.h +F: include/trace/events/hwmon*.h HARDWARE RANDOM NUMBER GENERATOR CORE M: Matt Mackall <mpm@selenic.com> @@ -8846,13 +8870,6 @@ S: Maintained F: Documentation/hwmon/max16065 F: drivers/hwmon/max16065.c -MAX20751 HARDWARE MONITOR DRIVER -M: Guenter Roeck <linux@roeck-us.net> -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/hwmon/max20751 -F: drivers/hwmon/max20751.c - MAX2175 SDR TUNER DRIVER M: Ramesh Shanmugasundaram <ramesh.shanmugasundaram@bp.renesas.com> L: linux-media@vger.kernel.org @@ -9698,6 +9715,19 @@ S: Maintained F: arch/arm/boot/dts/mmp* F: arch/arm/mach-mmp/ +MMU GATHER AND TLB INVALIDATION +M: Will Deacon <will.deacon@arm.com> +M: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com> +M: Andrew Morton <akpm@linux-foundation.org> +M: Nick Piggin <npiggin@gmail.com> +M: Peter Zijlstra <peterz@infradead.org> +L: linux-arch@vger.kernel.org +L: linux-mm@kvack.org +S: Maintained +F: arch/*/include/asm/tlb.h +F: include/asm-generic/tlb.h +F: mm/mmu_gather.c + MN88472 MEDIA DRIVER M: Antti Palosaari <crope@iki.fi> L: linux-media@vger.kernel.org @@ -10122,7 +10152,6 @@ L: netdev@vger.kernel.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec.git T: git git://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec-next.git S: Maintained -F: net/core/flow.c F: net/xfrm/ F: net/key/ F: net/ipv4/xfrm* @@ -11575,7 +11604,26 @@ W: http://hwmon.wiki.kernel.org/ W: http://www.roeck-us.net/linux/drivers/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git S: Maintained +F: Documentation/devicetree/bindings/hwmon/ibm,cffps1.txt +F: Documentation/devicetree/bindings/hwmon/max31785.txt +F: Documentation/devicetree/bindings/hwmon/ltc2978.txt +F: Documentation/hwmon/adm1275 +F: Documentation/hwmon/ibm-cffps +F: Documentation/hwmon/ir35221 +F: Documentation/hwmon/lm25066 +F: Documentation/hwmon/ltc2978 +F: Documentation/hwmon/ltc3815 +F: Documentation/hwmon/max16064 +F: Documentation/hwmon/max20751 +F: Documentation/hwmon/max31785 +F: Documentation/hwmon/max34440 +F: Documentation/hwmon/max8688 F: Documentation/hwmon/pmbus +F: Documentation/hwmon/pmbus-core +F: Documentation/hwmon/tps40422 +F: Documentation/hwmon/ucd9000 +F: Documentation/hwmon/ucd9200 +F: Documentation/hwmon/zl6100 F: drivers/hwmon/pmbus/ F: include/linux/pmbus.h @@ -13062,7 +13110,7 @@ SELINUX SECURITY MODULE M: Paul Moore <paul@paul-moore.com> M: Stephen Smalley <sds@tycho.nsa.gov> M: Eric Paris <eparis@parisplace.org> -L: selinux@tycho.nsa.gov (moderated for non-subscribers) +L: selinux@vger.kernel.org W: https://selinuxproject.org W: https://github.com/SELinuxProject T: git git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git @@ -13489,8 +13537,8 @@ L: linux-arm-kernel@lists.infradead.org S: Maintained F: Documentation/devicetree/bindings/arm/firmware/sdei.txt F: drivers/firmware/arm_sdei.c -F: include/linux/sdei.h -F: include/uapi/linux/sdei.h +F: include/linux/arm_sdei.h +F: include/uapi/linux/arm_sdei.h SOFTWARE RAID (Multiple Disks) SUPPORT M: Shaohua Li <shli@kernel.org> @@ -2,8 +2,8 @@ VERSION = 4 PATCHLEVEL = 19 SUBLEVEL = 0 -EXTRAVERSION = -rc8 -NAME = Merciless Moray +EXTRAVERSION = +NAME = "People's Front" # *DOCUMENTATION* # To see a list of typical targets execute "make help" diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index a045f3086047..e98c6b8e6186 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -9,6 +9,7 @@ config ARC def_bool y select ARC_TIMERS + select ARCH_HAS_DMA_COHERENT_TO_PFN select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE @@ -17,8 +18,7 @@ config ARC select BUILDTIME_EXTABLE_SORT select CLONE_BACKWARDS select COMMON_CLK - select DMA_NONCOHERENT_OPS - select DMA_NONCOHERENT_MMAP + select DMA_DIRECT_OPS select GENERIC_ATOMIC64 if !ISA_ARCV2 || !(ARC_HAS_LL64 && ARC_HAS_LLSC) select GENERIC_CLOCKEVENTS select GENERIC_FIND_FIRST_BIT diff --git a/arch/arc/mm/dma.c b/arch/arc/mm/dma.c index c75d5c3470e3..db203ff69ccf 100644 --- a/arch/arc/mm/dma.c +++ b/arch/arc/mm/dma.c @@ -84,29 +84,10 @@ void arch_dma_free(struct device *dev, size_t size, void *vaddr, __free_pages(page, get_order(size)); } -int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs) +long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr, + dma_addr_t dma_addr) { - unsigned long user_count = vma_pages(vma); - unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; - unsigned long pfn = __phys_to_pfn(dma_addr); - unsigned long off = vma->vm_pgoff; - int ret = -ENXIO; - - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - - if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) - return ret; - - if (off < count && user_count <= (count - off)) { - ret = remap_pfn_range(vma, vma->vm_start, - pfn + off, - user_count << PAGE_SHIFT, - vma->vm_page_prot); - } - - return ret; + return __phys_to_pfn(dma_addr); } /* @@ -167,7 +148,7 @@ void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, } /* - * Plug in coherent or noncoherent dma ops + * Plug in direct dma map ops. */ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, const struct iommu_ops *iommu, bool coherent) @@ -175,13 +156,11 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, /* * IOC hardware snoops all DMA traffic keeping the caches consistent * with memory - eliding need for any explicit cache maintenance of - * DMA buffers - so we can use dma_direct cache ops. + * DMA buffers. */ - if (is_isa_arcv2() && ioc_enable && coherent) { - set_dma_ops(dev, &dma_direct_ops); - dev_info(dev, "use dma_direct_ops cache ops\n"); - } else { - set_dma_ops(dev, &dma_noncoherent_ops); - dev_info(dev, "use dma_noncoherent_ops cache ops\n"); - } + if (is_isa_arcv2() && ioc_enable && coherent) + dev->dma_coherent = true; + + dev_info(dev, "use %sncoherent DMA ops\n", + dev->dma_coherent ? "" : "non"); } diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index 8436f6ade57d..965b7c846ecb 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -100,8 +100,10 @@ static inline unsigned long dma_max_pfn(struct device *dev) extern void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, const struct iommu_ops *iommu, bool coherent); +#ifdef CONFIG_MMU #define arch_teardown_dma_ops arch_teardown_dma_ops extern void arch_teardown_dma_ops(struct device *dev); +#endif /* do not use this function in a driver */ static inline bool is_device_dma_coherent(struct device *dev) diff --git a/arch/arm/include/asm/io.h b/arch/arm/include/asm/io.h index 2cfbc531f63b..6b51826ab3d1 100644 --- a/arch/arm/include/asm/io.h +++ b/arch/arm/include/asm/io.h @@ -28,7 +28,6 @@ #include <asm/byteorder.h> #include <asm/memory.h> #include <asm-generic/pci_iomap.h> -#include <xen/xen.h> /* * ISA I/O bus memory addresses are 1:1 with the physical address. @@ -459,20 +458,6 @@ extern void pci_iounmap(struct pci_dev *dev, void __iomem *addr); #include <asm-generic/io.h> -/* - * can the hardware map this into one segment or not, given no other - * constraints. - */ -#define BIOVEC_MERGEABLE(vec1, vec2) \ - ((bvec_to_phys((vec1)) + (vec1)->bv_len) == bvec_to_phys((vec2))) - -struct bio_vec; -extern bool xen_biovec_phys_mergeable(const struct bio_vec *vec1, - const struct bio_vec *vec2); -#define BIOVEC_PHYS_MERGEABLE(vec1, vec2) \ - (__BIOVEC_PHYS_MERGEABLE(vec1, vec2) && \ - (!xen_domain() || xen_biovec_phys_mergeable(vec1, vec2))) - #ifdef CONFIG_MMU #define ARCH_HAS_VALID_PHYS_ADDR_RANGE extern int valid_phys_addr_range(phys_addr_t addr, size_t size); diff --git a/arch/arm/include/asm/kvm_arm.h b/arch/arm/include/asm/kvm_arm.h index 3ab8b3781bfe..2d43dca29c72 100644 --- a/arch/arm/include/asm/kvm_arm.h +++ b/arch/arm/include/asm/kvm_arm.h @@ -161,6 +161,7 @@ #else #define VTTBR_X (5 - KVM_T0SZ) #endif +#define VTTBR_CNP_BIT _AC(1, UL) #define VTTBR_BADDR_MASK (((_AC(1, ULL) << (40 - VTTBR_X)) - 1) << VTTBR_X) #define VTTBR_VMID_SHIFT _AC(48, ULL) #define VTTBR_VMID_MASK(size) (_AT(u64, (1 << size) - 1) << VTTBR_VMID_SHIFT) diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h index 265ea9cf7df7..847f01fa429d 100644 --- a/arch/arm/include/asm/kvm_mmu.h +++ b/arch/arm/include/asm/kvm_mmu.h @@ -355,6 +355,11 @@ static inline int hyp_map_aux_data(void) #define kvm_phys_to_vttbr(addr) (addr) +static inline bool kvm_cpu_has_cnp(void) +{ + return false; +} + #endif /* !__ASSEMBLY__ */ #endif /* __ARM_KVM_MMU_H__ */ diff --git a/arch/arm/mach-ep93xx/snappercl15.c b/arch/arm/mach-ep93xx/snappercl15.c index 45940c1d7787..cf0cb58b3454 100644 --- a/arch/arm/mach-ep93xx/snappercl15.c +++ b/arch/arm/mach-ep93xx/snappercl15.c @@ -23,8 +23,7 @@ #include <linux/i2c.h> #include <linux/fb.h> -#include <linux/mtd/partitions.h> -#include <linux/mtd/rawnand.h> +#include <linux/mtd/platnand.h> #include <mach/hardware.h> #include <linux/platform_data/video-ep93xx.h> @@ -43,12 +42,11 @@ #define SNAPPERCL15_NAND_CEN (1 << 11) /* Chip enable (active low) */ #define SNAPPERCL15_NAND_RDY (1 << 14) /* Device ready */ -#define NAND_CTRL_ADDR(chip) (chip->IO_ADDR_W + 0x40) +#define NAND_CTRL_ADDR(chip) (chip->legacy.IO_ADDR_W + 0x40) -static void snappercl15_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, +static void snappercl15_nand_cmd_ctrl(struct nand_chip *chip, int cmd, unsigned int ctrl) { - struct nand_chip *chip = mtd_to_nand(mtd); static u16 nand_state = SNAPPERCL15_NAND_WPN; u16 set; @@ -70,13 +68,12 @@ static void snappercl15_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, } if (cmd != NAND_CMD_NONE) - __raw_writew((cmd & 0xff) | nand_state, chip->IO_ADDR_W); + __raw_writew((cmd & 0xff) | nand_state, + chip->legacy.IO_ADDR_W); } -static int snappercl15_nand_dev_ready(struct mtd_info *mtd) +static int snappercl15_nand_dev_ready(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); - return !!(__raw_readw(NAND_CTRL_ADDR(chip)) & SNAPPERCL15_NAND_RDY); } diff --git a/arch/arm/mach-ep93xx/ts72xx.c b/arch/arm/mach-ep93xx/ts72xx.c index c089a2a4fe30..c6a533699b00 100644 --- a/arch/arm/mach-ep93xx/ts72xx.c +++ b/arch/arm/mach-ep93xx/ts72xx.c @@ -16,8 +16,7 @@ #include <linux/init.h> #include <linux/platform_device.h> #include <linux/io.h> -#include <linux/mtd/rawnand.h> -#include <linux/mtd/partitions.h> +#include <linux/mtd/platnand.h> #include <linux/spi/spi.h> #include <linux/spi/flash.h> #include <linux/spi/mmc_spi.h> @@ -76,13 +75,11 @@ static void __init ts72xx_map_io(void) #define TS72XX_NAND_CONTROL_ADDR_LINE 22 /* 0xN0400000 */ #define TS72XX_NAND_BUSY_ADDR_LINE 23 /* 0xN0800000 */ -static void ts72xx_nand_hwcontrol(struct mtd_info *mtd, +static void ts72xx_nand_hwcontrol(struct nand_chip *chip, int cmd, unsigned int ctrl) { - struct nand_chip *chip = mtd_to_nand(mtd); - if (ctrl & NAND_CTRL_CHANGE) { - void __iomem *addr = chip->IO_ADDR_R; + void __iomem *addr = chip->legacy.IO_ADDR_R; unsigned char bits; addr += (1 << TS72XX_NAND_CONTROL_ADDR_LINE); @@ -96,13 +93,12 @@ static void ts72xx_nand_hwcontrol(struct mtd_info *mtd, } if (cmd != NAND_CMD_NONE) - __raw_writeb(cmd, chip->IO_ADDR_W); + __raw_writeb(cmd, chip->legacy.IO_ADDR_W); } -static int ts72xx_nand_device_ready(struct mtd_info *mtd) +static int ts72xx_nand_device_ready(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); - void __iomem *addr = chip->IO_ADDR_R; + void __iomem *addr = chip->legacy.IO_ADDR_R; addr += (1 << TS72XX_NAND_BUSY_ADDR_LINE); diff --git a/arch/arm/mach-imx/mach-qong.c b/arch/arm/mach-imx/mach-qong.c index 42a700053103..5c5df8ca38dd 100644 --- a/arch/arm/mach-imx/mach-qong.c +++ b/arch/arm/mach-imx/mach-qong.c @@ -18,7 +18,7 @@ #include <linux/memory.h> #include <linux/platform_device.h> #include <linux/mtd/physmap.h> -#include <linux/mtd/rawnand.h> +#include <linux/mtd/platnand.h> #include <linux/gpio.h> #include <asm/mach-types.h> @@ -129,30 +129,29 @@ static void qong_init_nor_mtd(void) /* * Hardware specific access to control-lines */ -static void qong_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl) +static void qong_nand_cmd_ctrl(struct nand_chip *nand_chip, int cmd, + unsigned int ctrl) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); - if (cmd == NAND_CMD_NONE) return; if (ctrl & NAND_CLE) - writeb(cmd, nand_chip->IO_ADDR_W + (1 << 24)); + writeb(cmd, nand_chip->legacy.IO_ADDR_W + (1 << 24)); else - writeb(cmd, nand_chip->IO_ADDR_W + (1 << 23)); + writeb(cmd, nand_chip->legacy.IO_ADDR_W + (1 << 23)); } /* * Read the Device Ready pin. */ -static int qong_nand_device_ready(struct mtd_info *mtd) +static int qong_nand_device_ready(struct nand_chip *chip) { return gpio_get_value(IOMUX_TO_GPIO(MX31_PIN_NFRB)); } -static void qong_nand_select_chip(struct mtd_info *mtd, int chip) +static void qong_nand_select_chip(struct nand_chip *chip, int cs) { - if (chip >= 0) + if (cs >= 0) gpio_set_value(IOMUX_TO_GPIO(MX31_PIN_NFCE_B), 0); else gpio_set_value(IOMUX_TO_GPIO(MX31_PIN_NFCE_B), 1); diff --git a/arch/arm/mach-ixp4xx/ixdp425-setup.c b/arch/arm/mach-ixp4xx/ixdp425-setup.c index 3ec829d52cdd..57d7df79d838 100644 --- a/arch/arm/mach-ixp4xx/ixdp425-setup.c +++ b/arch/arm/mach-ixp4xx/ixdp425-setup.c @@ -20,6 +20,7 @@ #include <linux/mtd/mtd.h> #include <linux/mtd/rawnand.h> #include <linux/mtd/partitions.h> +#include <linux/mtd/platnand.h> #include <linux/delay.h> #include <linux/gpio.h> #include <asm/types.h> @@ -75,9 +76,8 @@ static struct mtd_partition ixdp425_partitions[] = { }; static void -ixdp425_flash_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl) +ixdp425_flash_nand_cmd_ctrl(struct nand_chip *this, int cmd, unsigned int ctrl) { - struct nand_chip *this = mtd_to_nand(mtd); int offset = (int)nand_get_controller_data(this); if (ctrl & NAND_CTRL_CHANGE) { @@ -93,7 +93,7 @@ ixdp425_flash_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl) } if (cmd != NAND_CMD_NONE) - writeb(cmd, this->IO_ADDR_W + offset); + writeb(cmd, this->legacy.IO_ADDR_W + offset); } static struct platform_nand_data ixdp425_flash_nand_data = { diff --git a/arch/arm/mach-omap1/board-fsample.c b/arch/arm/mach-omap1/board-fsample.c index 69bd601feb83..4a0a66815ca0 100644 --- a/arch/arm/mach-omap1/board-fsample.c +++ b/arch/arm/mach-omap1/board-fsample.c @@ -16,8 +16,7 @@ #include <linux/platform_device.h> #include <linux/delay.h> #include <linux/mtd/mtd.h> -#include <linux/mtd/rawnand.h> -#include <linux/mtd/partitions.h> +#include <linux/mtd/platnand.h> #include <linux/mtd/physmap.h> #include <linux/input.h> #include <linux/smc91x.h> @@ -186,7 +185,7 @@ static struct platform_device nor_device = { #define FSAMPLE_NAND_RB_GPIO_PIN 62 -static int nand_dev_ready(struct mtd_info *mtd) +static int nand_dev_ready(struct nand_chip *chip) { return gpio_get_value(FSAMPLE_NAND_RB_GPIO_PIN); } diff --git a/arch/arm/mach-omap1/board-h2.c b/arch/arm/mach-omap1/board-h2.c index 9aeb8ad8c327..9d9a6ca15df0 100644 --- a/arch/arm/mach-omap1/board-h2.c +++ b/arch/arm/mach-omap1/board-h2.c @@ -24,8 +24,7 @@ #include <linux/delay.h> #include <linux/i2c.h> #include <linux/mtd/mtd.h> -#include <linux/mtd/rawnand.h> -#include <linux/mtd/partitions.h> +#include <linux/mtd/platnand.h> #include <linux/mtd/physmap.h> #include <linux/input.h> #include <linux/mfd/tps65010.h> @@ -182,7 +181,7 @@ static struct mtd_partition h2_nand_partitions[] = { #define H2_NAND_RB_GPIO_PIN 62 -static int h2_nand_dev_ready(struct mtd_info *mtd) +static int h2_nand_dev_ready(struct nand_chip *chip) { return gpio_get_value(H2_NAND_RB_GPIO_PIN); } diff --git a/arch/arm/mach-omap1/board-h3.c b/arch/arm/mach-omap1/board-h3.c index 2edcd6356f2d..cd6e02c5c01a 100644 --- a/arch/arm/mach-omap1/board-h3.c +++ b/arch/arm/mach-omap1/board-h3.c @@ -23,7 +23,7 @@ #include <linux/workqueue.h> #include <linux/i2c.h> #include <linux/mtd/mtd.h> -#include <linux/mtd/rawnand.h> +#include <linux/mtd/platnand.h> #include <linux/mtd/partitions.h> #include <linux/mtd/physmap.h> #include <linux/input.h> @@ -185,7 +185,7 @@ static struct mtd_partition nand_partitions[] = { #define H3_NAND_RB_GPIO_PIN 10 -static int nand_dev_ready(struct mtd_info *mtd) +static int nand_dev_ready(struct nand_chip *chip) { return gpio_get_value(H3_NAND_RB_GPIO_PIN); } diff --git a/arch/arm/mach-omap1/board-nand.c b/arch/arm/mach-omap1/board-nand.c index 1bffbb4e050f..20923eb2d9b6 100644 --- a/arch/arm/mach-omap1/board-nand.c +++ b/arch/arm/mach-omap1/board-nand.c @@ -20,9 +20,8 @@ #include "common.h" -void omap1_nand_cmd_ctl(struct mtd_info *mtd, int cmd, unsigned int ctrl) +void omap1_nand_cmd_ctl(struct nand_chip *this, int cmd, unsigned int ctrl) { - struct nand_chip *this = mtd_to_nand(mtd); unsigned long mask; if (cmd == NAND_CMD_NONE) @@ -32,6 +31,6 @@ void omap1_nand_cmd_ctl(struct mtd_info *mtd, int cmd, unsigned int ctrl) if (ctrl & NAND_ALE) mask |= 0x04; - writeb(cmd, this->IO_ADDR_W + mask); + writeb(cmd, this->legacy.IO_ADDR_W + mask); } diff --git a/arch/arm/mach-omap1/board-perseus2.c b/arch/arm/mach-omap1/board-perseus2.c index b4951eb82898..06a584fef5b8 100644 --- a/arch/arm/mach-omap1/board-perseus2.c +++ b/arch/arm/mach-omap1/board-perseus2.c @@ -16,8 +16,7 @@ #include <linux/platform_device.h> #include <linux/delay.h> #include <linux/mtd/mtd.h> -#include <linux/mtd/rawnand.h> -#include <linux/mtd/partitions.h> +#include <linux/mtd/platnand.h> #include <linux/mtd/physmap.h> #include <linux/input.h> #include <linux/smc91x.h> @@ -144,7 +143,7 @@ static struct platform_device nor_device = { #define P2_NAND_RB_GPIO_PIN 62 -static int nand_dev_ready(struct mtd_info *mtd) +static int nand_dev_ready(struct nand_chip *chip) { return gpio_get_value(P2_NAND_RB_GPIO_PIN); } diff --git a/arch/arm/mach-omap1/common.h b/arch/arm/mach-omap1/common.h index c6537d2c2859..504b959ba5cf 100644 --- a/arch/arm/mach-omap1/common.h +++ b/arch/arm/mach-omap1/common.h @@ -26,7 +26,6 @@ #ifndef __ARCH_ARM_MACH_OMAP1_COMMON_H #define __ARCH_ARM_MACH_OMAP1_COMMON_H -#include <linux/mtd/mtd.h> #include <linux/platform_data/i2c-omap.h> #include <linux/reboot.h> @@ -82,7 +81,8 @@ void omap1_restart(enum reboot_mode, const char *); extern void __init omap_check_revision(void); -extern void omap1_nand_cmd_ctl(struct mtd_info *mtd, int cmd, +struct nand_chip; +extern void omap1_nand_cmd_ctl(struct nand_chip *this, int cmd, unsigned int ctrl); extern void omap1_timer_init(void); diff --git a/arch/arm/mach-orion5x/ts78xx-setup.c b/arch/arm/mach-orion5x/ts78xx-setup.c index 94778739e38f..fda9b75c3a33 100644 --- a/arch/arm/mach-orion5x/ts78xx-setup.c +++ b/arch/arm/mach-orion5x/ts78xx-setup.c @@ -16,8 +16,7 @@ #include <linux/platform_device.h> #include <linux/mv643xx_eth.h> #include <linux/ata_platform.h> -#include <linux/mtd/rawnand.h> -#include <linux/mtd/partitions.h> +#include <linux/mtd/platnand.h> #include <linux/timeriomem-rng.h> #include <asm/mach-types.h> #include <asm/mach/arch.h> @@ -131,11 +130,9 @@ static void ts78xx_ts_rtc_unload(void) * NAND_CLE: bit 1 -> bit 1 * NAND_ALE: bit 2 -> bit 0 */ -static void ts78xx_ts_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, - unsigned int ctrl) +static void ts78xx_ts_nand_cmd_ctrl(struct nand_chip *this, int cmd, + unsigned int ctrl) { - struct nand_chip *this = mtd_to_nand(mtd); - if (ctrl & NAND_CTRL_CHANGE) { unsigned char bits; @@ -147,19 +144,18 @@ static void ts78xx_ts_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, } if (cmd != NAND_CMD_NONE) - writeb(cmd, this->IO_ADDR_W); + writeb(cmd, this->legacy.IO_ADDR_W); } -static int ts78xx_ts_nand_dev_ready(struct mtd_info *mtd) +static int ts78xx_ts_nand_dev_ready(struct nand_chip *chip) { return readb(TS_NAND_CTRL) & 0x20; } -static void ts78xx_ts_nand_write_buf(struct mtd_info *mtd, - const uint8_t *buf, int len) +static void ts78xx_ts_nand_write_buf(struct nand_chip *chip, + const uint8_t *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); - void __iomem *io_base = chip->IO_ADDR_W; + void __iomem *io_base = chip->legacy.IO_ADDR_W; unsigned long off = ((unsigned long)buf & 3); int sz; @@ -182,11 +178,10 @@ static void ts78xx_ts_nand_write_buf(struct mtd_info *mtd, writesb(io_base, buf, len); } -static void ts78xx_ts_nand_read_buf(struct mtd_info *mtd, - uint8_t *buf, int len) +static void ts78xx_ts_nand_read_buf(struct nand_chip *chip, + uint8_t *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); - void __iomem *io_base = chip->IO_ADDR_R; + void __iomem *io_base = chip->legacy.IO_ADDR_R; unsigned long off = ((unsigned long)buf & 3); int sz; diff --git a/arch/arm/mach-pxa/balloon3.c b/arch/arm/mach-pxa/balloon3.c index af46d2182533..c52c081eb6d9 100644 --- a/arch/arm/mach-pxa/balloon3.c +++ b/arch/arm/mach-pxa/balloon3.c @@ -25,11 +25,10 @@ #include <linux/ioport.h> #include <linux/ucb1400.h> #include <linux/mtd/mtd.h> -#include <linux/mtd/partitions.h> #include <linux/types.h> #include <linux/platform_data/pcf857x.h> #include <linux/platform_data/i2c-pxa.h> -#include <linux/mtd/rawnand.h> +#include <linux/mtd/platnand.h> #include <linux/mtd/physmap.h> #include <linux/regulator/max1586.h> @@ -571,9 +570,9 @@ static inline void balloon3_i2c_init(void) {} * NAND ******************************************************************************/ #if defined(CONFIG_MTD_NAND_PLATFORM)||defined(CONFIG_MTD_NAND_PLATFORM_MODULE) -static void balloon3_nand_cmd_ctl(struct mtd_info *mtd, int cmd, unsigned int ctrl) +static void balloon3_nand_cmd_ctl(struct nand_chip *this, int cmd, + unsigned int ctrl) { - struct nand_chip *this = mtd_to_nand(mtd); uint8_t balloon3_ctl_set = 0, balloon3_ctl_clr = 0; if (ctrl & NAND_CTRL_CHANGE) { @@ -597,10 +596,10 @@ static void balloon3_nand_cmd_ctl(struct mtd_info *mtd, int cmd, unsigned int ct } if (cmd != NAND_CMD_NONE) - writeb(cmd, this->IO_ADDR_W); + writeb(cmd, this->legacy.IO_ADDR_W); } -static void balloon3_nand_select_chip(struct mtd_info *mtd, int chip) +static void balloon3_nand_select_chip(struct nand_chip *this, int chip) { if (chip < 0 || chip > 3) return; @@ -616,7 +615,7 @@ static void balloon3_nand_select_chip(struct mtd_info *mtd, int chip) BALLOON3_NAND_CONTROL_REG); } -static int balloon3_nand_dev_ready(struct mtd_info *mtd) +static int balloon3_nand_dev_ready(struct nand_chip *this) { return __raw_readl(BALLOON3_NAND_STAT_REG) & BALLOON3_NAND_STAT_RNB; } diff --git a/arch/arm/mach-pxa/em-x270.c b/arch/arm/mach-pxa/em-x270.c index b14c47a6ee6b..67e37df637f5 100644 --- a/arch/arm/mach-pxa/em-x270.c +++ b/arch/arm/mach-pxa/em-x270.c @@ -15,8 +15,7 @@ #include <linux/dm9000.h> #include <linux/platform_data/rtc-v3020.h> -#include <linux/mtd/rawnand.h> -#include <linux/mtd/partitions.h> +#include <linux/mtd/platnand.h> #include <linux/mtd/physmap.h> #include <linux/input.h> #include <linux/gpio_keys.h> @@ -285,11 +284,10 @@ static void nand_cs_off(void) } /* hardware specific access to control-lines */ -static void em_x270_nand_cmd_ctl(struct mtd_info *mtd, int dat, +static void em_x270_nand_cmd_ctl(struct nand_chip *this, int dat, unsigned int ctrl) { - struct nand_chip *this = mtd_to_nand(mtd); - unsigned long nandaddr = (unsigned long)this->IO_ADDR_W; + unsigned long nandaddr = (unsigned long)this->legacy.IO_ADDR_W; dsb(); @@ -309,15 +307,15 @@ static void em_x270_nand_cmd_ctl(struct mtd_info *mtd, int dat, } dsb(); - this->IO_ADDR_W = (void __iomem *)nandaddr; + this->legacy.IO_ADDR_W = (void __iomem *)nandaddr; if (dat != NAND_CMD_NONE) - writel(dat, this->IO_ADDR_W); + writel(dat, this->legacy.IO_ADDR_W); dsb(); } /* read device ready pin */ -static int em_x270_nand_device_ready(struct mtd_info *mtd) +static int em_x270_nand_device_ready(struct nand_chip *this) { dsb(); diff --git a/arch/arm/mach-pxa/palmtreo.c b/arch/arm/mach-pxa/palmtreo.c index 4cc05ecce618..b66b0b11d717 100644 --- a/arch/arm/mach-pxa/palmtreo.c +++ b/arch/arm/mach-pxa/palmtreo.c @@ -404,36 +404,6 @@ static void __init palmtreo_leds_init(void) } /****************************************************************************** - * diskonchip docg4 flash - ******************************************************************************/ -#if defined(CONFIG_MACH_TREO680) -/* REVISIT: does the centro have this device also? */ -#if IS_ENABLED(CONFIG_MTD_NAND_DOCG4) -static struct resource docg4_resources[] = { - { - .start = 0x00000000, - .end = 0x00001FFF, - .flags = IORESOURCE_MEM, - }, -}; - -static struct platform_device treo680_docg4_flash = { - .name = "docg4", - .id = -1, - .resource = docg4_resources, - .num_resources = ARRAY_SIZE(docg4_resources), -}; - -static void __init treo680_docg4_flash_init(void) -{ - platform_device_register(&treo680_docg4_flash); -} -#else -static inline void treo680_docg4_flash_init(void) {} -#endif -#endif - -/****************************************************************************** * Machine init ******************************************************************************/ static void __init treo_reserve(void) @@ -517,7 +487,6 @@ static void __init treo680_init(void) treo680_gpio_init(); palm27x_mmc_init(GPIO_NR_TREO_SD_DETECT_N, GPIO_NR_TREO680_SD_READONLY, GPIO_NR_TREO680_SD_POWER, 0); - treo680_docg4_flash_init(); } #endif diff --git a/arch/arm/mach-pxa/palmtx.c b/arch/arm/mach-pxa/palmtx.c index 47e3e38e9bec..1d06a8e91d8f 100644 --- a/arch/arm/mach-pxa/palmtx.c +++ b/arch/arm/mach-pxa/palmtx.c @@ -28,8 +28,7 @@ #include <linux/wm97xx.h> #include <linux/power_supply.h> #include <linux/usb/gpio_vbus.h> -#include <linux/mtd/rawnand.h> -#include <linux/mtd/partitions.h> +#include <linux/mtd/platnand.h> #include <linux/mtd/mtd.h> #include <linux/mtd/physmap.h> @@ -247,11 +246,10 @@ static inline void palmtx_keys_init(void) {} ******************************************************************************/ #if defined(CONFIG_MTD_NAND_PLATFORM) || \ defined(CONFIG_MTD_NAND_PLATFORM_MODULE) -static void palmtx_nand_cmd_ctl(struct mtd_info *mtd, int cmd, - unsigned int ctrl) +static void palmtx_nand_cmd_ctl(struct nand_chip *this, int cmd, + unsigned int ctrl) { - struct nand_chip *this = mtd_to_nand(mtd); - char __iomem *nandaddr = this->IO_ADDR_W; + char __iomem *nandaddr = this->legacy.IO_ADDR_W; if (cmd == NAND_CMD_NONE) return; diff --git a/arch/arm/mm/dma-mapping-nommu.c b/arch/arm/mm/dma-mapping-nommu.c index f448a0663b10..712416ecd8e6 100644 --- a/arch/arm/mm/dma-mapping-nommu.c +++ b/arch/arm/mm/dma-mapping-nommu.c @@ -47,7 +47,8 @@ static void *arm_nommu_dma_alloc(struct device *dev, size_t size, */ if (attrs & DMA_ATTR_NON_CONSISTENT) - return dma_direct_alloc(dev, size, dma_handle, gfp, attrs); + return dma_direct_alloc_pages(dev, size, dma_handle, gfp, + attrs); ret = dma_alloc_from_global_coherent(size, dma_handle); @@ -70,7 +71,7 @@ static void arm_nommu_dma_free(struct device *dev, size_t size, unsigned long attrs) { if (attrs & DMA_ATTR_NON_CONSISTENT) { - dma_direct_free(dev, size, cpu_addr, dma_addr, attrs); + dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs); } else { int ret = dma_release_from_global_coherent(get_order(size), cpu_addr); @@ -90,7 +91,7 @@ static int arm_nommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret)) return ret; - return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size); + return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); } @@ -237,7 +238,3 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, set_dma_ops(dev, dma_ops); } - -void arch_teardown_dma_ops(struct device *dev) -{ -} diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 1b1a0e95c751..a8ae30fab508 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -75,6 +75,7 @@ config ARM64 select CLONE_BACKWARDS select COMMON_CLK select CPU_PM if (SUSPEND || CPU_IDLE) + select CRC32 select DCACHE_WORD_ACCESS select DMA_DIRECT_OPS select EDAC_SUPPORT @@ -142,6 +143,7 @@ config ARM64 select HAVE_PERF_USER_STACK_DUMP select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RCU_TABLE_FREE + select HAVE_RCU_TABLE_INVALIDATE select HAVE_RSEQ select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS @@ -479,6 +481,19 @@ config ARM64_ERRATUM_1024718 If unsure, say Y. +config ARM64_ERRATUM_1188873 + bool "Cortex-A76: MRC read following MRRC read of specific Generic Timer in AArch32 might give incorrect result" + default y + select ARM_ARCH_TIMER_OOL_WORKAROUND + help + This option adds work arounds for ARM Cortex-A76 erratum 1188873 + + Affected Cortex-A76 cores (r0p0, r1p0, r2p0) could cause + register corruption when accessing the timer registers from + AArch32 userspace. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y @@ -769,9 +784,6 @@ source kernel/Kconfig.hz config ARCH_SUPPORTS_DEBUG_PAGEALLOC def_bool y -config ARCH_HAS_HOLES_MEMORYMODEL - def_bool y if SPARSEMEM - config ARCH_SPARSEMEM_ENABLE def_bool y select SPARSEMEM_VMEMMAP_ENABLE @@ -786,7 +798,7 @@ config ARCH_FLATMEM_ENABLE def_bool !NUMA config HAVE_ARCH_PFN_VALID - def_bool ARCH_HAS_HOLES_MEMORYMODEL || !SPARSEMEM + def_bool y config HW_PERF_EVENTS def_bool y @@ -1132,6 +1144,20 @@ config ARM64_RAS_EXTN and access the new registers if the system supports the extension. Platform RAS features may additionally depend on firmware support. +config ARM64_CNP + bool "Enable support for Common Not Private (CNP) translations" + default y + depends on ARM64_PAN || !ARM64_SW_TTBR0_PAN + help + Common Not Private (CNP) allows translation table entries to + be shared between different PEs in the same inner shareable + domain, so the hardware can use this fact to optimise the + caching of such entries in the TLB. + + Selecting this option allows the CNP feature to be detected + at runtime, and does not affect PEs that do not implement + this feature. + endmenu config ARM64_SVE diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 0bcc98dbba56..6142402c2eb4 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -286,12 +286,11 @@ alternative_endif ldr \rd, [\rn, #MM_CONTEXT_ID] .endm /* - * read_ctr - read CTR_EL0. If the system has mismatched - * cache line sizes, provide the system wide safe value - * from arm64_ftr_reg_ctrel0.sys_val + * read_ctr - read CTR_EL0. If the system has mismatched register fields, + * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val */ .macro read_ctr, reg -alternative_if_not ARM64_MISMATCHED_CACHE_LINE_SIZE +alternative_if_not ARM64_MISMATCHED_CACHE_TYPE mrs \reg, ctr_el0 // read CTR nop alternative_else diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h index 5ee5bca8c24b..13dd42c3ad4e 100644 --- a/arch/arm64/include/asm/cache.h +++ b/arch/arm64/include/asm/cache.h @@ -40,6 +40,15 @@ #define L1_CACHE_SHIFT (6) #define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) + +#define CLIDR_LOUU_SHIFT 27 +#define CLIDR_LOC_SHIFT 24 +#define CLIDR_LOUIS_SHIFT 21 + +#define CLIDR_LOUU(clidr) (((clidr) >> CLIDR_LOUU_SHIFT) & 0x7) +#define CLIDR_LOC(clidr) (((clidr) >> CLIDR_LOC_SHIFT) & 0x7) +#define CLIDR_LOUIS(clidr) (((clidr) >> CLIDR_LOUIS_SHIFT) & 0x7) + /* * Memory returned by kmalloc() may be used for DMA, so we must make * sure that all such allocations are cache aligned. Otherwise, @@ -84,6 +93,37 @@ static inline int cache_line_size(void) return cwg ? 4 << cwg : ARCH_DMA_MINALIGN; } +/* + * Read the effective value of CTR_EL0. + * + * According to ARM ARM for ARMv8-A (ARM DDI 0487C.a), + * section D10.2.33 "CTR_EL0, Cache Type Register" : + * + * CTR_EL0.IDC reports the data cache clean requirements for + * instruction to data coherence. + * + * 0 - dcache clean to PoU is required unless : + * (CLIDR_EL1.LoC == 0) || (CLIDR_EL1.LoUIS == 0 && CLIDR_EL1.LoUU == 0) + * 1 - dcache clean to PoU is not required for i-to-d coherence. + * + * This routine provides the CTR_EL0 with the IDC field updated to the + * effective state. + */ +static inline u32 __attribute_const__ read_cpuid_effective_cachetype(void) +{ + u32 ctr = read_cpuid_cachetype(); + + if (!(ctr & BIT(CTR_IDC_SHIFT))) { + u64 clidr = read_sysreg(clidr_el1); + + if (CLIDR_LOC(clidr) == 0 || + (CLIDR_LOUIS(clidr) == 0 && CLIDR_LOUU(clidr) == 0)) + ctr |= BIT(CTR_IDC_SHIFT); + } + + return ctr; +} + #endif /* __ASSEMBLY__ */ #endif diff --git a/arch/arm64/include/asm/compat.h b/arch/arm64/include/asm/compat.h index 1a037b94eba1..cee28a05ee98 100644 --- a/arch/arm64/include/asm/compat.h +++ b/arch/arm64/include/asm/compat.h @@ -159,6 +159,7 @@ static inline compat_uptr_t ptr_to_compat(void __user *uptr) } #define compat_user_stack_pointer() (user_stack_pointer(task_pt_regs(current))) +#define COMPAT_MINSIGSTKSZ 2048 static inline void __user *arch_compat_alloc_user_space(long len) { diff --git a/arch/arm64/include/asm/compiler.h b/arch/arm64/include/asm/compiler.h deleted file mode 100644 index ee35fd0f2236..000000000000 --- a/arch/arm64/include/asm/compiler.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Based on arch/arm/include/asm/compiler.h - * - * Copyright (C) 2012 ARM Ltd. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ -#ifndef __ASM_COMPILER_H -#define __ASM_COMPILER_H - -/* - * This is used to ensure the compiler did actually allocate the register we - * asked it for some inline assembly sequences. Apparently we can't trust the - * compiler from one version to another so a bit of paranoia won't hurt. This - * string is meant to be concatenated with the inline asm string and will - * cause compilation to stop on mismatch. (for details, see gcc PR 15089) - */ -#define __asmeq(x, y) ".ifnc " x "," y " ; .err ; .endif\n\t" - -#endif /* __ASM_COMPILER_H */ diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index ae1f70450fb2..6e2d254c09eb 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -33,7 +33,7 @@ #define ARM64_WORKAROUND_CAVIUM_27456 12 #define ARM64_HAS_32BIT_EL0 13 #define ARM64_HARDEN_EL2_VECTORS 14 -#define ARM64_MISMATCHED_CACHE_LINE_SIZE 15 +#define ARM64_HAS_CNP 15 #define ARM64_HAS_NO_FPSIMD 16 #define ARM64_WORKAROUND_REPEAT_TLBI 17 #define ARM64_WORKAROUND_QCOM_FALKOR_E1003 18 @@ -51,7 +51,10 @@ #define ARM64_SSBD 30 #define ARM64_MISMATCHED_CACHE_TYPE 31 #define ARM64_HAS_STAGE2_FWB 32 +#define ARM64_HAS_CRC32 33 +#define ARM64_SSBS 34 +#define ARM64_WORKAROUND_1188873 35 -#define ARM64_NCAPS 33 +#define ARM64_NCAPS 36 #endif /* __ASM_CPUCAPS_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 1717ba1db35d..6db48d90ad63 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -262,7 +262,7 @@ extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0; /* * CPU feature detected at boot time based on system-wide value of a * feature. It is safe for a late CPU to have this feature even though - * the system hasn't enabled it, although the featuer will not be used + * the system hasn't enabled it, although the feature will not be used * by Linux in this case. If the system has enabled this feature already, * then every late CPU must have it. */ @@ -508,6 +508,12 @@ static inline bool system_supports_sve(void) cpus_have_const_cap(ARM64_SVE); } +static inline bool system_supports_cnp(void) +{ + return IS_ENABLED(CONFIG_ARM64_CNP) && + cpus_have_const_cap(ARM64_HAS_CNP); +} + #define ARM64_SSBD_UNKNOWN -1 #define ARM64_SSBD_FORCE_DISABLE 0 #define ARM64_SSBD_KERNEL 1 @@ -530,6 +536,7 @@ void arm64_set_ssbd_mitigation(bool state); static inline void arm64_set_ssbd_mitigation(bool state) {} #endif +extern int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt); #endif /* __ASSEMBLY__ */ #endif diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index ea690b3562af..12f93e4d2452 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -86,6 +86,7 @@ #define ARM_CPU_PART_CORTEX_A75 0xD0A #define ARM_CPU_PART_CORTEX_A35 0xD04 #define ARM_CPU_PART_CORTEX_A55 0xD05 +#define ARM_CPU_PART_CORTEX_A76 0xD0B #define APM_CPU_PART_POTENZA 0x000 @@ -110,6 +111,7 @@ #define MIDR_CORTEX_A75 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A75) #define MIDR_CORTEX_A35 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A35) #define MIDR_CORTEX_A55 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A55) +#define MIDR_CORTEX_A76 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) diff --git a/arch/arm64/include/asm/daifflags.h b/arch/arm64/include/asm/daifflags.h index 22e4c83de5a5..8d91f2233135 100644 --- a/arch/arm64/include/asm/daifflags.h +++ b/arch/arm64/include/asm/daifflags.h @@ -36,11 +36,8 @@ static inline unsigned long local_daif_save(void) { unsigned long flags; - asm volatile( - "mrs %0, daif // local_daif_save\n" - : "=r" (flags) - : - : "memory"); + flags = arch_local_save_flags(); + local_daif_mask(); return flags; @@ -60,11 +57,9 @@ static inline void local_daif_restore(unsigned long flags) { if (!arch_irqs_disabled_flags(flags)) trace_hardirqs_on(); - asm volatile( - "msr daif, %0 // local_daif_restore" - : - : "r" (flags) - : "memory"); + + arch_local_irq_restore(flags); + if (arch_irqs_disabled_flags(flags)) trace_hardirqs_off(); } diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index ce70c3ffb993..676de2ec1762 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -137,6 +137,8 @@ #define ESR_ELx_CV (UL(1) << 24) #define ESR_ELx_COND_SHIFT (20) #define ESR_ELx_COND_MASK (UL(0xF) << ESR_ELx_COND_SHIFT) +#define ESR_ELx_WFx_ISS_TI (UL(1) << 0) +#define ESR_ELx_WFx_ISS_WFI (UL(0) << 0) #define ESR_ELx_WFx_ISS_WFE (UL(1) << 0) #define ESR_ELx_xVC_IMM_MASK ((1UL << 16) - 1) @@ -148,6 +150,9 @@ #define DISR_EL1_ESR_MASK (ESR_ELx_AET | ESR_ELx_EA | ESR_ELx_FSC) /* ESR value templates for specific events */ +#define ESR_ELx_WFx_MASK (ESR_ELx_EC_MASK | ESR_ELx_WFx_ISS_TI) +#define ESR_ELx_WFx_WFI_VAL ((ESR_ELx_EC_WFx << ESR_ELx_EC_SHIFT) | \ + ESR_ELx_WFx_ISS_WFI) /* BRK instruction trap from AArch64 state */ #define ESR_ELx_VAL_BRK64(imm) \ @@ -187,6 +192,8 @@ #define ESR_ELx_SYS64_ISS_SYS_OP_MASK (ESR_ELx_SYS64_ISS_SYS_MASK | \ ESR_ELx_SYS64_ISS_DIR_MASK) +#define ESR_ELx_SYS64_ISS_RT(esr) \ + (((esr) & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT) /* * User space cache operations have the following sysreg encoding * in System instructions. @@ -206,6 +213,18 @@ #define ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL \ (ESR_ELx_SYS64_ISS_SYS_VAL(1, 3, 1, 7, 0) | \ ESR_ELx_SYS64_ISS_DIR_WRITE) +/* + * User space MRS operations which are supported for emulation + * have the following sysreg encoding in System instructions. + * op0 = 3, op1= 0, crn = 0, {crm = 0, 4-7}, READ (L = 1) + */ +#define ESR_ELx_SYS64_ISS_SYS_MRS_OP_MASK (ESR_ELx_SYS64_ISS_OP0_MASK | \ + ESR_ELx_SYS64_ISS_OP1_MASK | \ + ESR_ELx_SYS64_ISS_CRN_MASK | \ + ESR_ELx_SYS64_ISS_DIR_MASK) +#define ESR_ELx_SYS64_ISS_SYS_MRS_OP_VAL \ + (ESR_ELx_SYS64_ISS_SYS_VAL(3, 0, 0, 0, 0) | \ + ESR_ELx_SYS64_ISS_DIR_READ) #define ESR_ELx_SYS64_ISS_SYS_CTR ESR_ELx_SYS64_ISS_SYS_VAL(3, 3, 1, 0, 0) #define ESR_ELx_SYS64_ISS_SYS_CTR_READ (ESR_ELx_SYS64_ISS_SYS_CTR | \ @@ -249,6 +268,64 @@ #define ESR_ELx_FP_EXC_TFV (UL(1) << 23) +/* + * ISS field definitions for CP15 accesses + */ +#define ESR_ELx_CP15_32_ISS_DIR_MASK 0x1 +#define ESR_ELx_CP15_32_ISS_DIR_READ 0x1 +#define ESR_ELx_CP15_32_ISS_DIR_WRITE 0x0 + +#define ESR_ELx_CP15_32_ISS_RT_SHIFT 5 +#define ESR_ELx_CP15_32_ISS_RT_MASK (UL(0x1f) << ESR_ELx_CP15_32_ISS_RT_SHIFT) +#define ESR_ELx_CP15_32_ISS_CRM_SHIFT 1 +#define ESR_ELx_CP15_32_ISS_CRM_MASK (UL(0xf) << ESR_ELx_CP15_32_ISS_CRM_SHIFT) +#define ESR_ELx_CP15_32_ISS_CRN_SHIFT 10 +#define ESR_ELx_CP15_32_ISS_CRN_MASK (UL(0xf) << ESR_ELx_CP15_32_ISS_CRN_SHIFT) +#define ESR_ELx_CP15_32_ISS_OP1_SHIFT 14 +#define ESR_ELx_CP15_32_ISS_OP1_MASK (UL(0x7) << ESR_ELx_CP15_32_ISS_OP1_SHIFT) +#define ESR_ELx_CP15_32_ISS_OP2_SHIFT 17 +#define ESR_ELx_CP15_32_ISS_OP2_MASK (UL(0x7) << ESR_ELx_CP15_32_ISS_OP2_SHIFT) + +#define ESR_ELx_CP15_32_ISS_SYS_MASK (ESR_ELx_CP15_32_ISS_OP1_MASK | \ + ESR_ELx_CP15_32_ISS_OP2_MASK | \ + ESR_ELx_CP15_32_ISS_CRN_MASK | \ + ESR_ELx_CP15_32_ISS_CRM_MASK | \ + ESR_ELx_CP15_32_ISS_DIR_MASK) +#define ESR_ELx_CP15_32_ISS_SYS_VAL(op1, op2, crn, crm) \ + (((op1) << ESR_ELx_CP15_32_ISS_OP1_SHIFT) | \ + ((op2) << ESR_ELx_CP15_32_ISS_OP2_SHIFT) | \ + ((crn) << ESR_ELx_CP15_32_ISS_CRN_SHIFT) | \ + ((crm) << ESR_ELx_CP15_32_ISS_CRM_SHIFT)) + +#define ESR_ELx_CP15_64_ISS_DIR_MASK 0x1 +#define ESR_ELx_CP15_64_ISS_DIR_READ 0x1 +#define ESR_ELx_CP15_64_ISS_DIR_WRITE 0x0 + +#define ESR_ELx_CP15_64_ISS_RT_SHIFT 5 +#define ESR_ELx_CP15_64_ISS_RT_MASK (UL(0x1f) << ESR_ELx_CP15_64_ISS_RT_SHIFT) + +#define ESR_ELx_CP15_64_ISS_RT2_SHIFT 10 +#define ESR_ELx_CP15_64_ISS_RT2_MASK (UL(0x1f) << ESR_ELx_CP15_64_ISS_RT2_SHIFT) + +#define ESR_ELx_CP15_64_ISS_OP1_SHIFT 16 +#define ESR_ELx_CP15_64_ISS_OP1_MASK (UL(0xf) << ESR_ELx_CP15_64_ISS_OP1_SHIFT) +#define ESR_ELx_CP15_64_ISS_CRM_SHIFT 1 +#define ESR_ELx_CP15_64_ISS_CRM_MASK (UL(0xf) << ESR_ELx_CP15_64_ISS_CRM_SHIFT) + +#define ESR_ELx_CP15_64_ISS_SYS_VAL(op1, crm) \ + (((op1) << ESR_ELx_CP15_64_ISS_OP1_SHIFT) | \ + ((crm) << ESR_ELx_CP15_64_ISS_CRM_SHIFT)) + +#define ESR_ELx_CP15_64_ISS_SYS_MASK (ESR_ELx_CP15_64_ISS_OP1_MASK | \ + ESR_ELx_CP15_64_ISS_CRM_MASK | \ + ESR_ELx_CP15_64_ISS_DIR_MASK) + +#define ESR_ELx_CP15_64_ISS_SYS_CNTVCT (ESR_ELx_CP15_64_ISS_SYS_VAL(1, 14) | \ + ESR_ELx_CP15_64_ISS_DIR_READ) + +#define ESR_ELx_CP15_32_ISS_SYS_CNTFRQ (ESR_ELx_CP15_32_ISS_SYS_VAL(0, 0, 14, 0) |\ + ESR_ELx_CP15_32_ISS_DIR_READ) + #ifndef __ASSEMBLY__ #include <asm/types.h> diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 35b2e50f17fb..9f8b915af3a7 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -31,8 +31,6 @@ #include <asm/alternative.h> #include <asm/cpufeature.h> -#include <xen/xen.h> - /* * Generic IO read/write. These perform native-endian accesses. */ @@ -205,12 +203,5 @@ extern int valid_mmap_phys_addr_range(unsigned long pfn, size_t size); extern int devmem_is_allowed(unsigned long pfn); -struct bio_vec; -extern bool xen_biovec_phys_mergeable(const struct bio_vec *vec1, - const struct bio_vec *vec2); -#define BIOVEC_PHYS_MERGEABLE(vec1, vec2) \ - (__BIOVEC_PHYS_MERGEABLE(vec1, vec2) && \ - (!xen_domain() || xen_biovec_phys_mergeable(vec1, vec2))) - #endif /* __KERNEL__ */ #endif /* __ASM_IO_H */ diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h index a780f6714b44..850e2122d53f 100644 --- a/arch/arm64/include/asm/kernel-pgtable.h +++ b/arch/arm64/include/asm/kernel-pgtable.h @@ -97,7 +97,7 @@ + EARLY_PGDS((vstart), (vend)) /* each PGDIR needs a next level page table */ \ + EARLY_PUDS((vstart), (vend)) /* each PUD needs a next level page table */ \ + EARLY_PMDS((vstart), (vend))) /* each PMD needs a next level page table */ -#define SWAPPER_DIR_SIZE (PAGE_SIZE * EARLY_PAGES(KIMAGE_VADDR + TEXT_OFFSET, _end)) +#define INIT_DIR_SIZE (PAGE_SIZE * EARLY_PAGES(KIMAGE_VADDR + TEXT_OFFSET, _end)) #define IDMAP_DIR_SIZE (IDMAP_PGTABLE_LEVELS * PAGE_SIZE) #ifdef CONFIG_ARM64_SW_TTBR0_PAN diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index aa45df752a16..b476bc46f0ab 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -175,6 +175,7 @@ #define VTCR_EL2_FLAGS (VTCR_EL2_COMMON_BITS | VTCR_EL2_TGRAN_FLAGS) #define VTTBR_X (VTTBR_X_TGRAN_MAGIC - VTCR_EL2_T0SZ_IPA) +#define VTTBR_CNP_BIT (UL(1)) #define VTTBR_BADDR_MASK (((UL(1) << (PHYS_MASK_SHIFT - VTTBR_X)) - 1) << VTTBR_X) #define VTTBR_VMID_SHIFT (UL(48)) #define VTTBR_VMID_MASK(size) (_AT(u64, (1 << size) - 1) << VTTBR_VMID_SHIFT) diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 6106a85ae0be..21247870def7 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -335,7 +335,7 @@ static inline bool kvm_vcpu_dabt_isextabt(const struct kvm_vcpu *vcpu) static inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu) { u32 esr = kvm_vcpu_get_hsr(vcpu); - return (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT; + return ESR_ELx_SYS64_ISS_RT(esr); } static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 3d6d7336f871..2842bf149029 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -387,6 +387,8 @@ struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr); DECLARE_PER_CPU(kvm_cpu_context_t, kvm_host_cpu_state); +void __kvm_enable_ssbs(void); + static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr, unsigned long hyp_stack_ptr, unsigned long vector_ptr) @@ -407,6 +409,15 @@ static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr, */ BUG_ON(!static_branch_likely(&arm64_const_caps_ready)); __kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr, tpidr_el2); + + /* + * Disabling SSBD on a non-VHE system requires us to enable SSBS + * at EL2. + */ + if (!has_vhe() && this_cpu_has_cap(ARM64_SSBS) && + arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) { + kvm_call_hyp(__kvm_enable_ssbs); + } } static inline bool kvm_arch_check_sve_has_vhe(void) diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index d6fff7de5539..64337afbf124 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -517,5 +517,10 @@ static inline int hyp_map_aux_data(void) #define kvm_phys_to_vttbr(addr) phys_to_ttbr(addr) +static inline bool kvm_cpu_has_cnp(void) +{ + return system_supports_cnp(); +} + #endif /* __ASSEMBLY__ */ #endif /* __ARM64_KVM_MMU_H__ */ diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index dd320df0d026..7689c7aa1d77 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -95,5 +95,8 @@ extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, extern void *fixmap_remap_fdt(phys_addr_t dt_phys); extern void mark_linear_text_alias_ro(void); +#define INIT_MM_CONTEXT(name) \ + .pgd = init_pg_dir, + #endif /* !__ASSEMBLY__ */ #endif diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index 39ec0b8a689e..1e58bf58c22b 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -147,12 +147,25 @@ static inline void cpu_replace_ttbr1(pgd_t *pgdp) extern ttbr_replace_func idmap_cpu_replace_ttbr1; ttbr_replace_func *replace_phys; - phys_addr_t pgd_phys = virt_to_phys(pgdp); + /* phys_to_ttbr() zeros lower 2 bits of ttbr with 52-bit PA */ + phys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(pgdp)); + + if (system_supports_cnp() && !WARN_ON(pgdp != lm_alias(swapper_pg_dir))) { + /* + * cpu_replace_ttbr1() is used when there's a boot CPU + * up (i.e. cpufeature framework is not up yet) and + * latter only when we enable CNP via cpufeature's + * enable() callback. + * Also we rely on the cpu_hwcap bit being set before + * calling the enable() function. + */ + ttbr1 |= TTBR_CNP_BIT; + } replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1); cpu_install_idmap(); - replace_phys(pgd_phys); + replace_phys(ttbr1); cpu_uninstall_idmap(); } diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h index 60d02c81a3a2..c88a3cb117a1 100644 --- a/arch/arm64/include/asm/page.h +++ b/arch/arm64/include/asm/page.h @@ -37,9 +37,7 @@ extern void clear_page(void *to); typedef struct page *pgtable_t; -#ifdef CONFIG_HAVE_ARCH_PFN_VALID extern int pfn_valid(unsigned long); -#endif #include <asm/memory.h> diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h index fd208eac9f2a..1d7d8da2ef9b 100644 --- a/arch/arm64/include/asm/pgtable-hwdef.h +++ b/arch/arm64/include/asm/pgtable-hwdef.h @@ -211,6 +211,8 @@ #define PHYS_MASK_SHIFT (CONFIG_ARM64_PA_BITS) #define PHYS_MASK ((UL(1) << PHYS_MASK_SHIFT) - 1) +#define TTBR_CNP_BIT (UL(1) << 0) + /* * TCR flags. */ diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 1bdeca8918a6..50b1ef8584c0 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -360,6 +360,7 @@ static inline int pmd_protnone(pmd_t pmd) #define pmd_present(pmd) pte_present(pmd_pte(pmd)) #define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd)) #define pmd_young(pmd) pte_young(pmd_pte(pmd)) +#define pmd_valid(pmd) pte_valid(pmd_pte(pmd)) #define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd))) #define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd))) #define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd))) @@ -428,10 +429,33 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, PUD_TYPE_TABLE) #endif +extern pgd_t init_pg_dir[PTRS_PER_PGD]; +extern pgd_t init_pg_end[]; +extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; +extern pgd_t idmap_pg_dir[PTRS_PER_PGD]; +extern pgd_t tramp_pg_dir[PTRS_PER_PGD]; + +extern void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd); + +static inline bool in_swapper_pgdir(void *addr) +{ + return ((unsigned long)addr & PAGE_MASK) == + ((unsigned long)swapper_pg_dir & PAGE_MASK); +} + static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) { +#ifdef __PAGETABLE_PMD_FOLDED + if (in_swapper_pgdir(pmdp)) { + set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd))); + return; + } +#endif /* __PAGETABLE_PMD_FOLDED */ + WRITE_ONCE(*pmdp, pmd); - dsb(ishst); + + if (pmd_valid(pmd)) + dsb(ishst); } static inline void pmd_clear(pmd_t *pmdp) @@ -477,11 +501,21 @@ static inline phys_addr_t pmd_page_paddr(pmd_t pmd) #define pud_none(pud) (!pud_val(pud)) #define pud_bad(pud) (!(pud_val(pud) & PUD_TABLE_BIT)) #define pud_present(pud) pte_present(pud_pte(pud)) +#define pud_valid(pud) pte_valid(pud_pte(pud)) static inline void set_pud(pud_t *pudp, pud_t pud) { +#ifdef __PAGETABLE_PUD_FOLDED + if (in_swapper_pgdir(pudp)) { + set_swapper_pgd((pgd_t *)pudp, __pgd(pud_val(pud))); + return; + } +#endif /* __PAGETABLE_PUD_FOLDED */ + WRITE_ONCE(*pudp, pud); - dsb(ishst); + + if (pud_valid(pud)) + dsb(ishst); } static inline void pud_clear(pud_t *pudp) @@ -532,6 +566,11 @@ static inline phys_addr_t pud_page_paddr(pud_t pud) static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) { + if (in_swapper_pgdir(pgdp)) { + set_swapper_pgd(pgdp, pgd); + return; + } + WRITE_ONCE(*pgdp, pgd); dsb(ishst); } @@ -712,11 +751,6 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, } #endif -extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; -extern pgd_t swapper_pg_end[]; -extern pgd_t idmap_pg_dir[PTRS_PER_PGD]; -extern pgd_t tramp_pg_dir[PTRS_PER_PGD]; - /* * Encode and decode a swap entry: * bits 0-1: present (must be zero) diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 79657ad91397..2bf6691371c2 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -174,6 +174,10 @@ static inline void start_thread(struct pt_regs *regs, unsigned long pc, { start_thread_common(regs, pc); regs->pstate = PSR_MODE_EL0t; + + if (arm64_get_ssbd_state() != ARM64_SSBD_FORCE_ENABLE) + regs->pstate |= PSR_SSBS_BIT; + regs->sp = sp; } @@ -190,6 +194,9 @@ static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc, regs->pstate |= PSR_AA32_E_BIT; #endif + if (arm64_get_ssbd_state() != ARM64_SSBD_FORCE_ENABLE) + regs->pstate |= PSR_AA32_SSBS_BIT; + regs->compat_sp = sp; } #endif @@ -244,10 +251,6 @@ static inline void spin_lock_prefetch(const void *ptr) #endif -void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused); -void cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused); -void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused); - extern unsigned long __ro_after_init signal_minsigstksz; /* sigframe size */ extern void __init minsigstksz_setup(void); diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index 177b851ca6d9..6bc43889d11e 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -50,6 +50,7 @@ #define PSR_AA32_I_BIT 0x00000080 #define PSR_AA32_A_BIT 0x00000100 #define PSR_AA32_E_BIT 0x00000200 +#define PSR_AA32_SSBS_BIT 0x00800000 #define PSR_AA32_DIT_BIT 0x01000000 #define PSR_AA32_Q_BIT 0x08000000 #define PSR_AA32_V_BIT 0x10000000 diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index c1470931b897..0c909c4a932f 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -20,7 +20,6 @@ #ifndef __ASM_SYSREG_H #define __ASM_SYSREG_H -#include <asm/compiler.h> #include <linux/stringify.h> /* @@ -84,13 +83,26 @@ #endif /* CONFIG_BROKEN_GAS_INST */ -#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4) -#define REG_PSTATE_UAO_IMM sys_reg(0, 0, 4, 0, 3) +/* + * Instructions for modifying PSTATE fields. + * As per Arm ARM for v8-A, Section "C.5.1.3 op0 == 0b00, architectural hints, + * barriers and CLREX, and PSTATE access", ARM DDI 0487 C.a, system instructions + * for accessing PSTATE fields have the following encoding: + * Op0 = 0, CRn = 4 + * Op1, Op2 encodes the PSTATE field modified and defines the constraints. + * CRm = Imm4 for the instruction. + * Rt = 0x1f + */ +#define pstate_field(op1, op2) ((op1) << Op1_shift | (op2) << Op2_shift) +#define PSTATE_Imm_shift CRm_shift + +#define PSTATE_PAN pstate_field(0, 4) +#define PSTATE_UAO pstate_field(0, 3) +#define PSTATE_SSBS pstate_field(3, 1) -#define SET_PSTATE_PAN(x) __emit_inst(0xd5000000 | REG_PSTATE_PAN_IMM | \ - (!!x)<<8 | 0x1f) -#define SET_PSTATE_UAO(x) __emit_inst(0xd5000000 | REG_PSTATE_UAO_IMM | \ - (!!x)<<8 | 0x1f) +#define SET_PSTATE_PAN(x) __emit_inst(0xd500401f | PSTATE_PAN | ((!!x) << PSTATE_Imm_shift)) +#define SET_PSTATE_UAO(x) __emit_inst(0xd500401f | PSTATE_UAO | ((!!x) << PSTATE_Imm_shift)) +#define SET_PSTATE_SSBS(x) __emit_inst(0xd500401f | PSTATE_SSBS | ((!!x) << PSTATE_Imm_shift)) #define SYS_DC_ISW sys_insn(1, 0, 7, 6, 2) #define SYS_DC_CSW sys_insn(1, 0, 7, 10, 2) @@ -419,6 +431,7 @@ #define SYS_ICH_LR15_EL2 __SYS__LR8_EL2(7) /* Common SCTLR_ELx flags. */ +#define SCTLR_ELx_DSSBS (1UL << 44) #define SCTLR_ELx_EE (1 << 25) #define SCTLR_ELx_IESB (1 << 21) #define SCTLR_ELx_WXN (1 << 19) @@ -439,7 +452,7 @@ (1 << 10) | (1 << 13) | (1 << 14) | (1 << 15) | \ (1 << 17) | (1 << 20) | (1 << 24) | (1 << 26) | \ (1 << 27) | (1 << 30) | (1 << 31) | \ - (0xffffffffUL << 32)) + (0xffffefffUL << 32)) #ifdef CONFIG_CPU_BIG_ENDIAN #define ENDIAN_SET_EL2 SCTLR_ELx_EE @@ -453,7 +466,7 @@ #define SCTLR_EL2_SET (SCTLR_ELx_IESB | ENDIAN_SET_EL2 | SCTLR_EL2_RES1) #define SCTLR_EL2_CLEAR (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | \ SCTLR_ELx_SA | SCTLR_ELx_I | SCTLR_ELx_WXN | \ - ENDIAN_CLEAR_EL2 | SCTLR_EL2_RES0) + SCTLR_ELx_DSSBS | ENDIAN_CLEAR_EL2 | SCTLR_EL2_RES0) #if (SCTLR_EL2_SET ^ SCTLR_EL2_CLEAR) != 0xffffffffffffffff #error "Inconsistent SCTLR_EL2 set/clear bits" @@ -477,7 +490,7 @@ (1 << 29)) #define SCTLR_EL1_RES0 ((1 << 6) | (1 << 10) | (1 << 13) | (1 << 17) | \ (1 << 27) | (1 << 30) | (1 << 31) | \ - (0xffffffffUL << 32)) + (0xffffefffUL << 32)) #ifdef CONFIG_CPU_BIG_ENDIAN #define ENDIAN_SET_EL1 (SCTLR_EL1_E0E | SCTLR_ELx_EE) @@ -489,12 +502,12 @@ #define SCTLR_EL1_SET (SCTLR_ELx_M | SCTLR_ELx_C | SCTLR_ELx_SA |\ SCTLR_EL1_SA0 | SCTLR_EL1_SED | SCTLR_ELx_I |\ - SCTLR_EL1_DZE | SCTLR_EL1_UCT | SCTLR_EL1_NTWI |\ + SCTLR_EL1_DZE | SCTLR_EL1_UCT |\ SCTLR_EL1_NTWE | SCTLR_ELx_IESB | SCTLR_EL1_SPAN |\ ENDIAN_SET_EL1 | SCTLR_EL1_UCI | SCTLR_EL1_RES1) #define SCTLR_EL1_CLEAR (SCTLR_ELx_A | SCTLR_EL1_CP15BEN | SCTLR_EL1_ITD |\ SCTLR_EL1_UMA | SCTLR_ELx_WXN | ENDIAN_CLEAR_EL1 |\ - SCTLR_EL1_RES0) + SCTLR_ELx_DSSBS | SCTLR_EL1_NTWI | SCTLR_EL1_RES0) #if (SCTLR_EL1_SET ^ SCTLR_EL1_CLEAR) != 0xffffffffffffffff #error "Inconsistent SCTLR_EL1 set/clear bits" @@ -544,6 +557,13 @@ #define ID_AA64PFR0_EL0_64BIT_ONLY 0x1 #define ID_AA64PFR0_EL0_32BIT_64BIT 0x2 +/* id_aa64pfr1 */ +#define ID_AA64PFR1_SSBS_SHIFT 4 + +#define ID_AA64PFR1_SSBS_PSTATE_NI 0 +#define ID_AA64PFR1_SSBS_PSTATE_ONLY 1 +#define ID_AA64PFR1_SSBS_PSTATE_INSNS 2 + /* id_aa64mmfr0 */ #define ID_AA64MMFR0_TGRAN4_SHIFT 28 #define ID_AA64MMFR0_TGRAN64_SHIFT 24 diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index a3233167be60..106fdc951b6e 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -22,16 +22,10 @@ #include <linux/pagemap.h> #include <linux/swap.h> -#ifdef CONFIG_HAVE_RCU_TABLE_FREE - -#define tlb_remove_entry(tlb, entry) tlb_remove_table(tlb, entry) static inline void __tlb_remove_table(void *_table) { free_page_and_swap_cache((struct page *)_table); } -#else -#define tlb_remove_entry(tlb, entry) tlb_remove_page(tlb, entry) -#endif /* CONFIG_HAVE_RCU_TABLE_FREE */ static void tlb_flush(struct mmu_gather *tlb); @@ -40,36 +34,35 @@ static void tlb_flush(struct mmu_gather *tlb); static inline void tlb_flush(struct mmu_gather *tlb) { struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0); + bool last_level = !tlb->freed_tables; + unsigned long stride = tlb_get_unmap_size(tlb); /* - * The ASID allocator will either invalidate the ASID or mark - * it as used. + * If we're tearing down the address space then we only care about + * invalidating the walk-cache, since the ASID allocator won't + * reallocate our ASID without invalidating the entire TLB. */ - if (tlb->fullmm) + if (tlb->fullmm) { + if (!last_level) + flush_tlb_mm(tlb->mm); return; + } - /* - * The intermediate page table levels are already handled by - * the __(pte|pmd|pud)_free_tlb() functions, so last level - * TLBI is sufficient here. - */ - __flush_tlb_range(&vma, tlb->start, tlb->end, true); + __flush_tlb_range(&vma, tlb->start, tlb->end, stride, last_level); } static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr) { - __flush_tlb_pgtable(tlb->mm, addr); pgtable_page_dtor(pte); - tlb_remove_entry(tlb, pte); + tlb_remove_table(tlb, pte); } #if CONFIG_PGTABLE_LEVELS > 2 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr) { - __flush_tlb_pgtable(tlb->mm, addr); - tlb_remove_entry(tlb, virt_to_page(pmdp)); + tlb_remove_table(tlb, virt_to_page(pmdp)); } #endif @@ -77,8 +70,7 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, unsigned long addr) { - __flush_tlb_pgtable(tlb->mm, addr); - tlb_remove_entry(tlb, virt_to_page(pudp)); + tlb_remove_table(tlb, virt_to_page(pudp)); } #endif diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index a4a1901140ee..c3c0387aee18 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -70,43 +70,73 @@ }) /* - * TLB Management - * ============== + * TLB Invalidation + * ================ * - * The TLB specific code is expected to perform whatever tests it needs - * to determine if it should invalidate the TLB for each call. Start - * addresses are inclusive and end addresses are exclusive; it is safe to - * round these addresses down. + * This header file implements the low-level TLB invalidation routines + * (sometimes referred to as "flushing" in the kernel) for arm64. * - * flush_tlb_all() + * Every invalidation operation uses the following template: + * + * DSB ISHST // Ensure prior page-table updates have completed + * TLBI ... // Invalidate the TLB + * DSB ISH // Ensure the TLB invalidation has completed + * if (invalidated kernel mappings) + * ISB // Discard any instructions fetched from the old mapping + * + * + * The following functions form part of the "core" TLB invalidation API, + * as documented in Documentation/core-api/cachetlb.rst: * - * Invalidate the entire TLB. + * flush_tlb_all() + * Invalidate the entire TLB (kernel + user) on all CPUs * * flush_tlb_mm(mm) + * Invalidate an entire user address space on all CPUs. + * The 'mm' argument identifies the ASID to invalidate. + * + * flush_tlb_range(vma, start, end) + * Invalidate the virtual-address range '[start, end)' on all + * CPUs for the user address space corresponding to 'vma->mm'. + * Note that this operation also invalidates any walk-cache + * entries associated with translations for the specified address + * range. + * + * flush_tlb_kernel_range(start, end) + * Same as flush_tlb_range(..., start, end), but applies to + * kernel mappings rather than a particular user address space. + * Whilst not explicitly documented, this function is used when + * unmapping pages from vmalloc/io space. + * + * flush_tlb_page(vma, addr) + * Invalidate a single user mapping for address 'addr' in the + * address space corresponding to 'vma->mm'. Note that this + * operation only invalidates a single, last-level page-table + * entry and therefore does not affect any walk-caches. * - * Invalidate all TLB entries in a particular address space. - * - mm - mm_struct describing address space * - * flush_tlb_range(mm,start,end) + * Next, we have some undocumented invalidation routines that you probably + * don't want to call unless you know what you're doing: * - * Invalidate a range of TLB entries in the specified address - * space. - * - mm - mm_struct describing address space - * - start - start address (may not be aligned) - * - end - end address (exclusive, may not be aligned) + * local_flush_tlb_all() + * Same as flush_tlb_all(), but only applies to the calling CPU. * - * flush_tlb_page(vaddr,vma) + * __flush_tlb_kernel_pgtable(addr) + * Invalidate a single kernel mapping for address 'addr' on all + * CPUs, ensuring that any walk-cache entries associated with the + * translation are also invalidated. * - * Invalidate the specified page in the specified address range. - * - vaddr - virtual address (may not be aligned) - * - vma - vma_struct describing address range + * __flush_tlb_range(vma, start, end, stride, last_level) + * Invalidate the virtual-address range '[start, end)' on all + * CPUs for the user address space corresponding to 'vma->mm'. + * The invalidation operations are issued at a granularity + * determined by 'stride' and only affect any walk-cache entries + * if 'last_level' is equal to false. * - * flush_kern_tlb_page(kaddr) * - * Invalidate the TLB entry for the specified page. The address - * will be in the kernels virtual memory space. Current uses - * only require the D-TLB to be invalidated. - * - kaddr - Kernel virtual memory address + * Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented + * on top of these routines, since that is our interface to the mmu_gather + * API as used by munmap() and friends. */ static inline void local_flush_tlb_all(void) { @@ -149,25 +179,28 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, * This is meant to avoid soft lock-ups on large TLB flushing ranges and not * necessarily a performance improvement. */ -#define MAX_TLB_RANGE (1024UL << PAGE_SHIFT) +#define MAX_TLBI_OPS 1024UL static inline void __flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, - bool last_level) + unsigned long stride, bool last_level) { unsigned long asid = ASID(vma->vm_mm); unsigned long addr; - if ((end - start) > MAX_TLB_RANGE) { + if ((end - start) > (MAX_TLBI_OPS * stride)) { flush_tlb_mm(vma->vm_mm); return; } + /* Convert the stride into units of 4k */ + stride >>= 12; + start = __TLBI_VADDR(start, asid); end = __TLBI_VADDR(end, asid); dsb(ishst); - for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) { + for (addr = start; addr < end; addr += stride) { if (last_level) { __tlbi(vale1is, addr); __tlbi_user(vale1is, addr); @@ -182,14 +215,18 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma, static inline void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - __flush_tlb_range(vma, start, end, false); + /* + * We cannot use leaf-only invalidation here, since we may be invalidating + * table entries as part of collapsing hugepages or moving page tables. + */ + __flush_tlb_range(vma, start, end, PAGE_SIZE, false); } static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end) { unsigned long addr; - if ((end - start) > MAX_TLB_RANGE) { + if ((end - start) > (MAX_TLBI_OPS * PAGE_SIZE)) { flush_tlb_all(); return; } @@ -199,7 +236,7 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end dsb(ishst); for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) - __tlbi(vaae1is, addr); + __tlbi(vaale1is, addr); dsb(ish); isb(); } @@ -208,20 +245,11 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end * Used to invalidate the TLB (walk caches) corresponding to intermediate page * table levels (pgd/pud/pmd). */ -static inline void __flush_tlb_pgtable(struct mm_struct *mm, - unsigned long uaddr) -{ - unsigned long addr = __TLBI_VADDR(uaddr, ASID(mm)); - - __tlbi(vae1is, addr); - __tlbi_user(vae1is, addr); - dsb(ish); -} - static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr) { unsigned long addr = __TLBI_VADDR(kaddr, 0); + dsb(ishst); __tlbi(vaae1is, addr); dsb(ish); } diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index e66b0fca99c2..07c34087bd5e 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -32,7 +32,6 @@ #include <asm/cpufeature.h> #include <asm/ptrace.h> #include <asm/memory.h> -#include <asm/compiler.h> #include <asm/extable.h> #define get_ds() (KERNEL_DS) diff --git a/arch/arm64/include/asm/xen/events.h b/arch/arm64/include/asm/xen/events.h index 4e22b7a8c038..2788e95d0ff0 100644 --- a/arch/arm64/include/asm/xen/events.h +++ b/arch/arm64/include/asm/xen/events.h @@ -14,7 +14,7 @@ enum ipi_vector { static inline int xen_irqs_disabled(struct pt_regs *regs) { - return raw_irqs_disabled_flags((unsigned long) regs->pstate); + return !interrupts_enabled(regs); } #define xchg_xen_ulong(ptr, val) xchg((ptr), (val)) diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 17c65c8f33cb..2bcd6e4f3474 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -48,5 +48,6 @@ #define HWCAP_USCAT (1 << 25) #define HWCAP_ILRCPC (1 << 26) #define HWCAP_FLAGM (1 << 27) +#define HWCAP_SSBS (1 << 28) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi/asm/ptrace.h index 98c4ce55d9c3..a36227fdb084 100644 --- a/arch/arm64/include/uapi/asm/ptrace.h +++ b/arch/arm64/include/uapi/asm/ptrace.h @@ -46,6 +46,7 @@ #define PSR_I_BIT 0x00000080 #define PSR_A_BIT 0x00000100 #define PSR_D_BIT 0x00000200 +#define PSR_SSBS_BIT 0x00001000 #define PSR_PAN_BIT 0x00400000 #define PSR_UAO_BIT 0x00800000 #define PSR_V_BIT 0x10000000 diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index dec10898d688..a509e35132d2 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -68,21 +68,43 @@ static bool has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry, int scope) { - u64 mask = CTR_CACHE_MINLINE_MASK; - - /* Skip matching the min line sizes for cache type check */ - if (entry->capability == ARM64_MISMATCHED_CACHE_TYPE) - mask ^= arm64_ftr_reg_ctrel0.strict_mask; + u64 mask = arm64_ftr_reg_ctrel0.strict_mask; + u64 sys = arm64_ftr_reg_ctrel0.sys_val & mask; + u64 ctr_raw, ctr_real; WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); - return (read_cpuid_cachetype() & mask) != - (arm64_ftr_reg_ctrel0.sys_val & mask); + + /* + * We want to make sure that all the CPUs in the system expose + * a consistent CTR_EL0 to make sure that applications behaves + * correctly with migration. + * + * If a CPU has CTR_EL0.IDC but does not advertise it via CTR_EL0 : + * + * 1) It is safe if the system doesn't support IDC, as CPU anyway + * reports IDC = 0, consistent with the rest. + * + * 2) If the system has IDC, it is still safe as we trap CTR_EL0 + * access on this CPU via the ARM64_HAS_CACHE_IDC capability. + * + * So, we need to make sure either the raw CTR_EL0 or the effective + * CTR_EL0 matches the system's copy to allow a secondary CPU to boot. + */ + ctr_raw = read_cpuid_cachetype() & mask; + ctr_real = read_cpuid_effective_cachetype() & mask; + + return (ctr_real != sys) && (ctr_raw != sys); } static void cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *__unused) { - sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0); + u64 mask = arm64_ftr_reg_ctrel0.strict_mask; + + /* Trap CTR_EL0 access on this CPU, only if it has a mismatch */ + if ((read_cpuid_cachetype() & mask) != + (arm64_ftr_reg_ctrel0.sys_val & mask)) + sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0); } atomic_t arm64_el2_vector_last_slot = ATOMIC_INIT(-1); @@ -116,6 +138,15 @@ static void __install_bp_hardening_cb(bp_hardening_cb_t fn, static DEFINE_SPINLOCK(bp_lock); int cpu, slot = -1; + /* + * enable_smccc_arch_workaround_1() passes NULL for the hyp_vecs + * start/end if we're a guest. Skip the hyp-vectors work. + */ + if (!hyp_vecs_start) { + __this_cpu_write(bp_hardening_data.fn, fn); + return; + } + spin_lock(&bp_lock); for_each_possible_cpu(cpu) { if (per_cpu(bp_hardening_data.fn, cpu) == fn) { @@ -312,6 +343,14 @@ void __init arm64_enable_wa2_handling(struct alt_instr *alt, void arm64_set_ssbd_mitigation(bool state) { + if (this_cpu_has_cap(ARM64_SSBS)) { + if (state) + asm volatile(SET_PSTATE_SSBS(0)); + else + asm volatile(SET_PSTATE_SSBS(1)); + return; + } + switch (psci_ops.conduit) { case PSCI_CONDUIT_HVC: arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_2, state, NULL); @@ -336,6 +375,11 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry, WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); + if (this_cpu_has_cap(ARM64_SSBS)) { + required = false; + goto out_printmsg; + } + if (psci_ops.smccc_version == SMCCC_VERSION_1_0) { ssbd_state = ARM64_SSBD_UNKNOWN; return false; @@ -384,7 +428,6 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry, switch (ssbd_state) { case ARM64_SSBD_FORCE_DISABLE: - pr_info_once("%s disabled from command-line\n", entry->desc); arm64_set_ssbd_mitigation(false); required = false; break; @@ -397,7 +440,6 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry, break; case ARM64_SSBD_FORCE_ENABLE: - pr_info_once("%s forced from command-line\n", entry->desc); arm64_set_ssbd_mitigation(true); required = true; break; @@ -407,10 +449,27 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry, break; } +out_printmsg: + switch (ssbd_state) { + case ARM64_SSBD_FORCE_DISABLE: + pr_info_once("%s disabled from command-line\n", entry->desc); + break; + + case ARM64_SSBD_FORCE_ENABLE: + pr_info_once("%s forced from command-line\n", entry->desc); + break; + } + return required; } #endif /* CONFIG_ARM64_SSBD */ +static void __maybe_unused +cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused) +{ + sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCI, 0); +} + #define CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max) \ .matches = is_affected_midr_range, \ .midr_range = MIDR_RANGE(model, v_min, r_min, v_max, r_max) @@ -616,14 +675,7 @@ const struct arm64_cpu_capabilities arm64_errata[] = { }, #endif { - .desc = "Mismatched cache line size", - .capability = ARM64_MISMATCHED_CACHE_LINE_SIZE, - .matches = has_mismatched_cache_type, - .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, - .cpu_enable = cpu_enable_trap_ctr_access, - }, - { - .desc = "Mismatched cache type", + .desc = "Mismatched cache type (CTR_EL0)", .capability = ARM64_MISMATCHED_CACHE_TYPE, .matches = has_mismatched_cache_type, .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, @@ -680,6 +732,14 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .matches = has_ssbd_mitigation, }, #endif +#ifdef CONFIG_ARM64_ERRATUM_1188873 + { + /* Cortex-A76 r0p0 to r2p0 */ + .desc = "ARM erratum 1188873", + .capability = ARM64_WORKAROUND_1188873, + ERRATA_MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 2, 0), + }, +#endif { } }; diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index e238b7932096..af50064dea51 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -20,6 +20,7 @@ #include <linux/bsearch.h> #include <linux/cpumask.h> +#include <linux/crash_dump.h> #include <linux/sort.h> #include <linux/stop_machine.h> #include <linux/types.h> @@ -117,6 +118,7 @@ EXPORT_SYMBOL(cpu_hwcap_keys); static bool __maybe_unused cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused); +static void cpu_enable_cnp(struct arm64_cpu_capabilities const *cap); /* * NOTE: Any changes to the visibility of features should be kept in @@ -164,6 +166,11 @@ static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = { ARM64_FTR_END, }; +static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = { + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_SSBS_SHIFT, 4, ID_AA64PFR1_SSBS_PSTATE_NI), + ARM64_FTR_END, +}; + static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = { S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN4_SHIFT, 4, ID_AA64MMFR0_TGRAN4_NI), S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN64_SHIFT, 4, ID_AA64MMFR0_TGRAN64_NI), @@ -371,7 +378,7 @@ static const struct __ftr_reg_entry { /* Op1 = 0, CRn = 0, CRm = 4 */ ARM64_FTR_REG(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0), - ARM64_FTR_REG(SYS_ID_AA64PFR1_EL1, ftr_raz), + ARM64_FTR_REG(SYS_ID_AA64PFR1_EL1, ftr_id_aa64pfr1), ARM64_FTR_REG(SYS_ID_AA64ZFR0_EL1, ftr_raz), /* Op1 = 0, CRn = 0, CRm = 5 */ @@ -657,7 +664,6 @@ void update_cpu_features(int cpu, /* * EL3 is not our concern. - * ID_AA64PFR1 is currently RES0. */ taint |= check_update_ftr_reg(SYS_ID_AA64PFR0_EL1, cpu, info->reg_id_aa64pfr0, boot->reg_id_aa64pfr0); @@ -848,15 +854,55 @@ static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unus } static bool has_cache_idc(const struct arm64_cpu_capabilities *entry, - int __unused) + int scope) { - return read_sanitised_ftr_reg(SYS_CTR_EL0) & BIT(CTR_IDC_SHIFT); + u64 ctr; + + if (scope == SCOPE_SYSTEM) + ctr = arm64_ftr_reg_ctrel0.sys_val; + else + ctr = read_cpuid_effective_cachetype(); + + return ctr & BIT(CTR_IDC_SHIFT); +} + +static void cpu_emulate_effective_ctr(const struct arm64_cpu_capabilities *__unused) +{ + /* + * If the CPU exposes raw CTR_EL0.IDC = 0, while effectively + * CTR_EL0.IDC = 1 (from CLIDR values), we need to trap accesses + * to the CTR_EL0 on this CPU and emulate it with the real/safe + * value. + */ + if (!(read_cpuid_cachetype() & BIT(CTR_IDC_SHIFT))) + sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0); } static bool has_cache_dic(const struct arm64_cpu_capabilities *entry, - int __unused) + int scope) { - return read_sanitised_ftr_reg(SYS_CTR_EL0) & BIT(CTR_DIC_SHIFT); + u64 ctr; + + if (scope == SCOPE_SYSTEM) + ctr = arm64_ftr_reg_ctrel0.sys_val; + else + ctr = read_cpuid_cachetype(); + + return ctr & BIT(CTR_DIC_SHIFT); +} + +static bool __maybe_unused +has_useable_cnp(const struct arm64_cpu_capabilities *entry, int scope) +{ + /* + * Kdump isn't guaranteed to power-off all secondary CPUs, CNP + * may share TLB entries with a CPU stuck in the crashed + * kernel. + */ + if (is_kdump_kernel()) + return false; + + return has_cpuid_feature(entry, scope); } #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 @@ -1035,6 +1081,70 @@ static void cpu_has_fwb(const struct arm64_cpu_capabilities *__unused) WARN_ON(val & (7 << 27 | 7 << 21)); } +#ifdef CONFIG_ARM64_SSBD +static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr) +{ + if (user_mode(regs)) + return 1; + + if (instr & BIT(PSTATE_Imm_shift)) + regs->pstate |= PSR_SSBS_BIT; + else + regs->pstate &= ~PSR_SSBS_BIT; + + arm64_skip_faulting_instruction(regs, 4); + return 0; +} + +static struct undef_hook ssbs_emulation_hook = { + .instr_mask = ~(1U << PSTATE_Imm_shift), + .instr_val = 0xd500401f | PSTATE_SSBS, + .fn = ssbs_emulation_handler, +}; + +static void cpu_enable_ssbs(const struct arm64_cpu_capabilities *__unused) +{ + static bool undef_hook_registered = false; + static DEFINE_SPINLOCK(hook_lock); + + spin_lock(&hook_lock); + if (!undef_hook_registered) { + register_undef_hook(&ssbs_emulation_hook); + undef_hook_registered = true; + } + spin_unlock(&hook_lock); + + if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) { + sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS); + arm64_set_ssbd_mitigation(false); + } else { + arm64_set_ssbd_mitigation(true); + } +} +#endif /* CONFIG_ARM64_SSBD */ + +#ifdef CONFIG_ARM64_PAN +static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) +{ + /* + * We modify PSTATE. This won't work from irq context as the PSTATE + * is discarded once we return from the exception. + */ + WARN_ON_ONCE(in_interrupt()); + + sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0); + asm(SET_PSTATE_PAN(1)); +} +#endif /* CONFIG_ARM64_PAN */ + +#ifdef CONFIG_ARM64_RAS_EXTN +static void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused) +{ + /* Firmware may have left a deferred SError in this register. */ + write_sysreg_s(0, SYS_DISR_EL1); +} +#endif /* CONFIG_ARM64_RAS_EXTN */ + static const struct arm64_cpu_capabilities arm64_features[] = { { .desc = "GIC system register CPU interface", @@ -1184,6 +1294,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .capability = ARM64_HAS_CACHE_IDC, .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_cache_idc, + .cpu_enable = cpu_emulate_effective_ctr, }, { .desc = "Instruction cache invalidation not required for I/D coherence", @@ -1222,6 +1333,41 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = cpu_enable_hw_dbm, }, #endif +#ifdef CONFIG_ARM64_SSBD + { + .desc = "CRC32 instructions", + .capability = ARM64_HAS_CRC32, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + .sys_reg = SYS_ID_AA64ISAR0_EL1, + .field_pos = ID_AA64ISAR0_CRC32_SHIFT, + .min_field_value = 1, + }, + { + .desc = "Speculative Store Bypassing Safe (SSBS)", + .capability = ARM64_SSBS, + .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE, + .matches = has_cpuid_feature, + .sys_reg = SYS_ID_AA64PFR1_EL1, + .field_pos = ID_AA64PFR1_SSBS_SHIFT, + .sign = FTR_UNSIGNED, + .min_field_value = ID_AA64PFR1_SSBS_PSTATE_ONLY, + .cpu_enable = cpu_enable_ssbs, + }, +#endif +#ifdef CONFIG_ARM64_CNP + { + .desc = "Common not Private translations", + .capability = ARM64_HAS_CNP, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_useable_cnp, + .sys_reg = SYS_ID_AA64MMFR2_EL1, + .sign = FTR_UNSIGNED, + .field_pos = ID_AA64MMFR2_CNP_SHIFT, + .min_field_value = 1, + .cpu_enable = cpu_enable_cnp, + }, +#endif {}, }; @@ -1267,6 +1413,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { #ifdef CONFIG_ARM64_SVE HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, HWCAP_SVE), #endif + HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_SSBS_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_SSBS_PSTATE_INSNS, CAP_HWCAP, HWCAP_SSBS), {}, }; @@ -1658,6 +1805,11 @@ cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused) return (cpus_have_const_cap(ARM64_HAS_PAN) && !cpus_have_const_cap(ARM64_HAS_UAO)); } +static void __maybe_unused cpu_enable_cnp(struct arm64_cpu_capabilities const *cap) +{ + cpu_replace_ttbr1(lm_alias(swapper_pg_dir)); +} + /* * We emulate only the following system register space. * Op0 = 0x3, CRn = 0x0, Op1 = 0x0, CRm = [0, 4 - 7] @@ -1719,27 +1871,32 @@ static int emulate_sys_reg(u32 id, u64 *valp) return 0; } -static int emulate_mrs(struct pt_regs *regs, u32 insn) +int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt) { int rc; - u32 sys_reg, dst; u64 val; - /* - * sys_reg values are defined as used in mrs/msr instruction. - * shift the imm value to get the encoding. - */ - sys_reg = (u32)aarch64_insn_decode_immediate(AARCH64_INSN_IMM_16, insn) << 5; rc = emulate_sys_reg(sys_reg, &val); if (!rc) { - dst = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn); - pt_regs_write_reg(regs, dst, val); + pt_regs_write_reg(regs, rt, val); arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); } - return rc; } +static int emulate_mrs(struct pt_regs *regs, u32 insn) +{ + u32 sys_reg, rt; + + /* + * sys_reg values are defined as used in mrs/msr instruction. + * shift the imm value to get the encoding. + */ + sys_reg = (u32)aarch64_insn_decode_immediate(AARCH64_INSN_IMM_16, insn) << 5; + rt = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn); + return do_emulate_mrs(regs, sys_reg, rt); +} + static struct undef_hook mrs_hook = { .instr_mask = 0xfff00000, .instr_val = 0xd5300000, @@ -1755,9 +1912,3 @@ static int __init enable_mrs_emulation(void) } core_initcall(enable_mrs_emulation); - -void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused) -{ - /* Firmware may have left a deferred SError in this register. */ - write_sysreg_s(0, SYS_DISR_EL1); -} diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index e9ab7b3ed317..bcc2831399cb 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -81,6 +81,7 @@ static const char *const hwcap_str[] = { "uscat", "ilrcpc", "flagm", + "ssbs", NULL }; @@ -324,7 +325,15 @@ static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info) static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) { info->reg_cntfrq = arch_timer_get_cntfrq(); - info->reg_ctr = read_cpuid_cachetype(); + /* + * Use the effective value of the CTR_EL0 than the raw value + * exposed by the CPU. CTR_E0.IDC field value must be interpreted + * with the CLIDR_EL1 fields to avoid triggering false warnings + * when there is a mismatch across the CPUs. Keep track of the + * effective value of the CTR_EL0 in our internal records for + * acurate sanity check and feature enablement. + */ + info->reg_ctr = read_cpuid_effective_cachetype(); info->reg_dczid = read_cpuid(DCZID_EL0); info->reg_midr = read_cpuid_id(); info->reg_revidr = read_cpuid(REVIDR_EL1); diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 09dbea221a27..039144ecbcb2 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -589,7 +589,7 @@ el1_undef: inherit_daif pstate=x23, tmp=x2 mov x0, sp bl do_undefinstr - ASM_BUG() + kernel_exit 1 el1_dbg: /* * Debug exception handling @@ -665,6 +665,7 @@ el0_sync: cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception b.eq el0_fpsimd_exc cmp x24, #ESR_ELx_EC_SYS64 // configurable trap + ccmp x24, #ESR_ELx_EC_WFx, #4, ne b.eq el0_sys cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception b.eq el0_sp_pc @@ -697,9 +698,9 @@ el0_sync_compat: cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0 b.eq el0_undef cmp x24, #ESR_ELx_EC_CP15_32 // CP15 MRC/MCR trap - b.eq el0_undef + b.eq el0_cp15 cmp x24, #ESR_ELx_EC_CP15_64 // CP15 MRRC/MCRR trap - b.eq el0_undef + b.eq el0_cp15 cmp x24, #ESR_ELx_EC_CP14_MR // CP14 MRC/MCR trap b.eq el0_undef cmp x24, #ESR_ELx_EC_CP14_LS // CP14 LDC/STC trap @@ -722,6 +723,17 @@ el0_irq_compat: el0_error_compat: kernel_entry 0, 32 b el0_error_naked + +el0_cp15: + /* + * Trapped CP15 (MRC, MCR, MRRC, MCRR) instructions + */ + enable_daif + ct_user_exit + mov x0, x25 + mov x1, sp + bl do_cp15instr + b ret_to_user #endif el0_da: diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index b0853069702f..4471f570a295 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -287,19 +287,21 @@ __create_page_tables: mov x28, lr /* - * Invalidate the idmap and swapper page tables to avoid potential - * dirty cache lines being evicted. + * Invalidate the init page tables to avoid potential dirty cache lines + * being evicted. Other page tables are allocated in rodata as part of + * the kernel image, and thus are clean to the PoC per the boot + * protocol. */ - adrp x0, idmap_pg_dir - adrp x1, swapper_pg_end + adrp x0, init_pg_dir + adrp x1, init_pg_end sub x1, x1, x0 bl __inval_dcache_area /* - * Clear the idmap and swapper page tables. + * Clear the init page tables. */ - adrp x0, idmap_pg_dir - adrp x1, swapper_pg_end + adrp x0, init_pg_dir + adrp x1, init_pg_end sub x1, x1, x0 1: stp xzr, xzr, [x0], #16 stp xzr, xzr, [x0], #16 @@ -373,7 +375,7 @@ __create_page_tables: /* * Map the kernel image (starting with PHYS_OFFSET). */ - adrp x0, swapper_pg_dir + adrp x0, init_pg_dir mov_q x5, KIMAGE_VADDR + TEXT_OFFSET // compile time __va(_text) add x5, x5, x23 // add KASLR displacement mov x4, PTRS_PER_PGD @@ -390,7 +392,7 @@ __create_page_tables: * tables again to remove any speculatively loaded cache lines. */ adrp x0, idmap_pg_dir - adrp x1, swapper_pg_end + adrp x1, init_pg_end sub x1, x1, x0 dmb sy bl __inval_dcache_area @@ -706,6 +708,7 @@ secondary_startup: * Common entry point for secondary CPUs. */ bl __cpu_setup // initialise processor + adrp x1, swapper_pg_dir bl __enable_mmu ldr x8, =__secondary_switched br x8 @@ -748,6 +751,7 @@ ENDPROC(__secondary_switched) * Enable the MMU. * * x0 = SCTLR_EL1 value for turning on the MMU. + * x1 = TTBR1_EL1 value * * Returns to the caller via x30/lr. This requires the caller to be covered * by the .idmap.text section. @@ -756,17 +760,16 @@ ENDPROC(__secondary_switched) * If it isn't, park the CPU */ ENTRY(__enable_mmu) - mrs x1, ID_AA64MMFR0_EL1 - ubfx x2, x1, #ID_AA64MMFR0_TGRAN_SHIFT, 4 + mrs x2, ID_AA64MMFR0_EL1 + ubfx x2, x2, #ID_AA64MMFR0_TGRAN_SHIFT, 4 cmp x2, #ID_AA64MMFR0_TGRAN_SUPPORTED b.ne __no_granule_support - update_early_cpu_boot_status 0, x1, x2 - adrp x1, idmap_pg_dir - adrp x2, swapper_pg_dir - phys_to_ttbr x3, x1 - phys_to_ttbr x4, x2 - msr ttbr0_el1, x3 // load TTBR0 - msr ttbr1_el1, x4 // load TTBR1 + update_early_cpu_boot_status 0, x2, x3 + adrp x2, idmap_pg_dir + phys_to_ttbr x1, x1 + phys_to_ttbr x2, x2 + msr ttbr0_el1, x2 // load TTBR0 + msr ttbr1_el1, x1 // load TTBR1 isb msr sctlr_el1, x0 isb @@ -823,6 +826,7 @@ __primary_switch: mrs x20, sctlr_el1 // preserve old SCTLR_EL1 value #endif + adrp x1, init_pg_dir bl __enable_mmu #ifdef CONFIG_RELOCATABLE bl __relocate_kernel diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c index e78c3ef04d95..9b65132e789a 100644 --- a/arch/arm64/kernel/probes/kprobes.c +++ b/arch/arm64/kernel/probes/kprobes.c @@ -107,7 +107,7 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) if (!p->ainsn.api.insn) return -ENOMEM; break; - }; + } /* prepare the instruction */ if (p->ainsn.api.insn) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 7f1628effe6d..ce99c58cd1f1 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -358,6 +358,10 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, if (IS_ENABLED(CONFIG_ARM64_UAO) && cpus_have_const_cap(ARM64_HAS_UAO)) childregs->pstate |= PSR_UAO_BIT; + + if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) + childregs->pstate |= PSR_SSBS_BIT; + p->thread.cpu_context.x19 = stack_start; p->thread.cpu_context.x20 = stk_sz; } diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index e8edbf13302a..8cdaf25e99cd 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -24,7 +24,6 @@ #include <uapi/linux/psci.h> -#include <asm/compiler.h> #include <asm/cpu_ops.h> #include <asm/errno.h> #include <asm/smp_plat.h> diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index b3354ff94e79..d0f62dd24c90 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -349,12 +349,8 @@ void __init setup_arch(char **cmdline_p) #endif #ifdef CONFIG_VT -#if defined(CONFIG_VGA_CONSOLE) - conswitchp = &vga_con; -#elif defined(CONFIG_DUMMY_CONSOLE) conswitchp = &dummy_con; #endif -#endif if (boot_args[1] || boot_args[2] || boot_args[3]) { pr_err("WARNING: x1-x3 nonzero in violation of boot protocol:\n" "\tx1: %016llx\n\tx2: %016llx\n\tx3: %016llx\n" diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S index bebec8ef9372..3e53ffa07994 100644 --- a/arch/arm64/kernel/sleep.S +++ b/arch/arm64/kernel/sleep.S @@ -101,6 +101,7 @@ ENTRY(cpu_resume) bl el2_setup // if in EL2 drop to EL1 cleanly bl __cpu_setup /* enable the MMU early - so we can access sleep_save_stash by va */ + adrp x1, swapper_pg_dir bl __enable_mmu ldr x8, =_cpu_resume br x8 diff --git a/arch/arm64/kernel/ssbd.c b/arch/arm64/kernel/ssbd.c index 3432e5ef9f41..885f13e58708 100644 --- a/arch/arm64/kernel/ssbd.c +++ b/arch/arm64/kernel/ssbd.c @@ -3,17 +3,33 @@ * Copyright (C) 2018 ARM Ltd, All Rights Reserved. */ +#include <linux/compat.h> #include <linux/errno.h> #include <linux/sched.h> +#include <linux/sched/task_stack.h> #include <linux/thread_info.h> #include <asm/cpufeature.h> +static void ssbd_ssbs_enable(struct task_struct *task) +{ + u64 val = is_compat_thread(task_thread_info(task)) ? + PSR_AA32_SSBS_BIT : PSR_SSBS_BIT; + + task_pt_regs(task)->pstate |= val; +} + +static void ssbd_ssbs_disable(struct task_struct *task) +{ + u64 val = is_compat_thread(task_thread_info(task)) ? + PSR_AA32_SSBS_BIT : PSR_SSBS_BIT; + + task_pt_regs(task)->pstate &= ~val; +} + /* * prctl interface for SSBD - * FIXME: Drop the below ifdefery once merged in 4.18. */ -#ifdef PR_SPEC_STORE_BYPASS static int ssbd_prctl_set(struct task_struct *task, unsigned long ctrl) { int state = arm64_get_ssbd_state(); @@ -46,12 +62,14 @@ static int ssbd_prctl_set(struct task_struct *task, unsigned long ctrl) return -EPERM; task_clear_spec_ssb_disable(task); clear_tsk_thread_flag(task, TIF_SSBD); + ssbd_ssbs_enable(task); break; case PR_SPEC_DISABLE: if (state == ARM64_SSBD_FORCE_DISABLE) return -EPERM; task_set_spec_ssb_disable(task); set_tsk_thread_flag(task, TIF_SSBD); + ssbd_ssbs_disable(task); break; case PR_SPEC_FORCE_DISABLE: if (state == ARM64_SSBD_FORCE_DISABLE) @@ -59,6 +77,7 @@ static int ssbd_prctl_set(struct task_struct *task, unsigned long ctrl) task_set_spec_ssb_disable(task); task_set_spec_ssb_force_disable(task); set_tsk_thread_flag(task, TIF_SSBD); + ssbd_ssbs_disable(task); break; default: return -ERANGE; @@ -107,4 +126,3 @@ int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) return -ENODEV; } } -#endif /* PR_SPEC_STORE_BYPASS */ diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index 70c283368b64..9405d1b7f4b0 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -48,6 +48,10 @@ void notrace __cpu_suspend_exit(void) */ cpu_uninstall_idmap(); + /* Restore CnP bit in TTBR1_EL1 */ + if (system_supports_cnp()) + cpu_replace_ttbr1(lm_alias(swapper_pg_dir)); + /* * PSTATE was not saved over suspend/resume, re-enable any detected * features that might not have been set correctly. diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 039e9ff379cc..4066da7f1e5e 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -310,10 +310,12 @@ static int call_undef_hook(struct pt_regs *regs) int (*fn)(struct pt_regs *regs, u32 instr) = NULL; void __user *pc = (void __user *)instruction_pointer(regs); - if (!user_mode(regs)) - return 1; - - if (compat_thumb_mode(regs)) { + if (!user_mode(regs)) { + __le32 instr_le; + if (probe_kernel_address((__force __le32 *)pc, instr_le)) + goto exit; + instr = le32_to_cpu(instr_le); + } else if (compat_thumb_mode(regs)) { /* 16-bit Thumb instruction */ __le16 instr_le; if (get_user(instr_le, (__le16 __user *)pc)) @@ -352,6 +354,9 @@ void force_signal_inject(int signal, int code, unsigned long address) const char *desc; struct pt_regs *regs = current_pt_regs(); + if (WARN_ON(!user_mode(regs))) + return; + clear_siginfo(&info); switch (signal) { @@ -406,14 +411,10 @@ asmlinkage void __exception do_undefinstr(struct pt_regs *regs) if (call_undef_hook(regs) == 0) return; + BUG_ON(!user_mode(regs)); force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); } -void cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused) -{ - sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCI, 0); -} - #define __user_cache_maint(insn, address, res) \ if (address >= user_addr_max()) { \ res = -EFAULT; \ @@ -437,7 +438,7 @@ void cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused) static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs) { unsigned long address; - int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT; + int rt = ESR_ELx_SYS64_ISS_RT(esr); int crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT; int ret = 0; @@ -472,7 +473,7 @@ static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs) static void ctr_read_handler(unsigned int esr, struct pt_regs *regs) { - int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT; + int rt = ESR_ELx_SYS64_ISS_RT(esr); unsigned long val = arm64_ftr_reg_user_value(&arm64_ftr_reg_ctrel0); pt_regs_write_reg(regs, rt, val); @@ -482,7 +483,7 @@ static void ctr_read_handler(unsigned int esr, struct pt_regs *regs) static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs) { - int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT; + int rt = ESR_ELx_SYS64_ISS_RT(esr); pt_regs_write_reg(regs, rt, arch_counter_get_cntvct()); arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); @@ -490,12 +491,28 @@ static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs) static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs) { - int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT; + int rt = ESR_ELx_SYS64_ISS_RT(esr); pt_regs_write_reg(regs, rt, arch_timer_get_rate()); arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); } +static void mrs_handler(unsigned int esr, struct pt_regs *regs) +{ + u32 sysreg, rt; + + rt = ESR_ELx_SYS64_ISS_RT(esr); + sysreg = esr_sys64_to_sysreg(esr); + + if (do_emulate_mrs(regs, sysreg, rt) != 0) + force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); +} + +static void wfi_handler(unsigned int esr, struct pt_regs *regs) +{ + arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); +} + struct sys64_hook { unsigned int esr_mask; unsigned int esr_val; @@ -526,9 +543,176 @@ static struct sys64_hook sys64_hooks[] = { .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTFRQ, .handler = cntfrq_read_handler, }, + { + /* Trap read access to CPUID registers */ + .esr_mask = ESR_ELx_SYS64_ISS_SYS_MRS_OP_MASK, + .esr_val = ESR_ELx_SYS64_ISS_SYS_MRS_OP_VAL, + .handler = mrs_handler, + }, + { + /* Trap WFI instructions executed in userspace */ + .esr_mask = ESR_ELx_WFx_MASK, + .esr_val = ESR_ELx_WFx_WFI_VAL, + .handler = wfi_handler, + }, {}, }; + +#ifdef CONFIG_COMPAT +#define PSTATE_IT_1_0_SHIFT 25 +#define PSTATE_IT_1_0_MASK (0x3 << PSTATE_IT_1_0_SHIFT) +#define PSTATE_IT_7_2_SHIFT 10 +#define PSTATE_IT_7_2_MASK (0x3f << PSTATE_IT_7_2_SHIFT) + +static u32 compat_get_it_state(struct pt_regs *regs) +{ + u32 it, pstate = regs->pstate; + + it = (pstate & PSTATE_IT_1_0_MASK) >> PSTATE_IT_1_0_SHIFT; + it |= ((pstate & PSTATE_IT_7_2_MASK) >> PSTATE_IT_7_2_SHIFT) << 2; + + return it; +} + +static void compat_set_it_state(struct pt_regs *regs, u32 it) +{ + u32 pstate_it; + + pstate_it = (it << PSTATE_IT_1_0_SHIFT) & PSTATE_IT_1_0_MASK; + pstate_it |= ((it >> 2) << PSTATE_IT_7_2_SHIFT) & PSTATE_IT_7_2_MASK; + + regs->pstate &= ~PSR_AA32_IT_MASK; + regs->pstate |= pstate_it; +} + +static bool cp15_cond_valid(unsigned int esr, struct pt_regs *regs) +{ + int cond; + + /* Only a T32 instruction can trap without CV being set */ + if (!(esr & ESR_ELx_CV)) { + u32 it; + + it = compat_get_it_state(regs); + if (!it) + return true; + + cond = it >> 4; + } else { + cond = (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT; + } + + return aarch32_opcode_cond_checks[cond](regs->pstate); +} + +static void advance_itstate(struct pt_regs *regs) +{ + u32 it; + + /* ARM mode */ + if (!(regs->pstate & PSR_AA32_T_BIT) || + !(regs->pstate & PSR_AA32_IT_MASK)) + return; + + it = compat_get_it_state(regs); + + /* + * If this is the last instruction of the block, wipe the IT + * state. Otherwise advance it. + */ + if (!(it & 7)) + it = 0; + else + it = (it & 0xe0) | ((it << 1) & 0x1f); + + compat_set_it_state(regs, it); +} + +static void arm64_compat_skip_faulting_instruction(struct pt_regs *regs, + unsigned int sz) +{ + advance_itstate(regs); + arm64_skip_faulting_instruction(regs, sz); +} + +static void compat_cntfrq_read_handler(unsigned int esr, struct pt_regs *regs) +{ + int reg = (esr & ESR_ELx_CP15_32_ISS_RT_MASK) >> ESR_ELx_CP15_32_ISS_RT_SHIFT; + + pt_regs_write_reg(regs, reg, arch_timer_get_rate()); + arm64_compat_skip_faulting_instruction(regs, 4); +} + +static struct sys64_hook cp15_32_hooks[] = { + { + .esr_mask = ESR_ELx_CP15_32_ISS_SYS_MASK, + .esr_val = ESR_ELx_CP15_32_ISS_SYS_CNTFRQ, + .handler = compat_cntfrq_read_handler, + }, + {}, +}; + +static void compat_cntvct_read_handler(unsigned int esr, struct pt_regs *regs) +{ + int rt = (esr & ESR_ELx_CP15_64_ISS_RT_MASK) >> ESR_ELx_CP15_64_ISS_RT_SHIFT; + int rt2 = (esr & ESR_ELx_CP15_64_ISS_RT2_MASK) >> ESR_ELx_CP15_64_ISS_RT2_SHIFT; + u64 val = arch_counter_get_cntvct(); + + pt_regs_write_reg(regs, rt, lower_32_bits(val)); + pt_regs_write_reg(regs, rt2, upper_32_bits(val)); + arm64_compat_skip_faulting_instruction(regs, 4); +} + +static struct sys64_hook cp15_64_hooks[] = { + { + .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK, + .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCT, + .handler = compat_cntvct_read_handler, + }, + {}, +}; + +asmlinkage void __exception do_cp15instr(unsigned int esr, struct pt_regs *regs) +{ + struct sys64_hook *hook, *hook_base; + + if (!cp15_cond_valid(esr, regs)) { + /* + * There is no T16 variant of a CP access, so we + * always advance PC by 4 bytes. + */ + arm64_compat_skip_faulting_instruction(regs, 4); + return; + } + + switch (ESR_ELx_EC(esr)) { + case ESR_ELx_EC_CP15_32: + hook_base = cp15_32_hooks; + break; + case ESR_ELx_EC_CP15_64: + hook_base = cp15_64_hooks; + break; + default: + do_undefinstr(regs); + return; + } + + for (hook = hook_base; hook->handler; hook++) + if ((hook->esr_mask & esr) == hook->esr_val) { + hook->handler(esr, regs); + return; + } + + /* + * New cp15 instructions may previously have been undefined at + * EL0. Fall back to our usual undefined instruction handler + * so that we handle these consistently. + */ + do_undefinstr(regs); +} +#endif + asmlinkage void __exception do_sysinstr(unsigned int esr, struct pt_regs *regs) { struct sys64_hook *hook; @@ -605,7 +789,6 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) handler[reason], smp_processor_id(), esr, esr_get_class_string(esr)); - die("Oops - bad mode", regs, 0); local_daif_mask(); panic("bad mode"); } diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 605d1b60469c..ab29c06a7d4b 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -138,6 +138,23 @@ SECTIONS EXCEPTION_TABLE(8) /* __init_begin will be marked RO NX */ NOTES + . = ALIGN(PAGE_SIZE); + idmap_pg_dir = .; + . += IDMAP_DIR_SIZE; + +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 + tramp_pg_dir = .; + . += PAGE_SIZE; +#endif + +#ifdef CONFIG_ARM64_SW_TTBR0_PAN + reserved_ttbr0 = .; + . += RESERVED_TTBR0_SIZE; +#endif + swapper_pg_dir = .; + . += PAGE_SIZE; + swapper_pg_end = .; + . = ALIGN(SEGMENT_ALIGN); __init_begin = .; __inittext_begin = .; @@ -216,21 +233,9 @@ SECTIONS BSS_SECTION(0, 0, 0) . = ALIGN(PAGE_SIZE); - idmap_pg_dir = .; - . += IDMAP_DIR_SIZE; - -#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 - tramp_pg_dir = .; - . += PAGE_SIZE; -#endif - -#ifdef CONFIG_ARM64_SW_TTBR0_PAN - reserved_ttbr0 = .; - . += RESERVED_TTBR0_SIZE; -#endif - swapper_pg_dir = .; - . += SWAPPER_DIR_SIZE; - swapper_pg_end = .; + init_pg_dir = .; + . += INIT_DIR_SIZE; + init_pg_end = .; __pecoff_data_size = ABSOLUTE(. - __initdata_begin); _end = .; diff --git a/arch/arm64/kvm/hyp-init.S b/arch/arm64/kvm/hyp-init.S index ea9225160786..4576b86a5579 100644 --- a/arch/arm64/kvm/hyp-init.S +++ b/arch/arm64/kvm/hyp-init.S @@ -65,6 +65,9 @@ __do_hyp_init: b.lo __kvm_handle_stub_hvc phys_to_ttbr x4, x0 +alternative_if ARM64_HAS_CNP + orr x4, x4, #TTBR_CNP_BIT +alternative_else_nop_endif msr ttbr0_el2, x4 mrs x4, tcr_el1 diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c index 9ce223944983..76d016b446b2 100644 --- a/arch/arm64/kvm/hyp/sysreg-sr.c +++ b/arch/arm64/kvm/hyp/sysreg-sr.c @@ -288,3 +288,14 @@ void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu) vcpu->arch.sysregs_loaded_on_cpu = false; } + +void __hyp_text __kvm_enable_ssbs(void) +{ + u64 tmp; + + asm volatile( + "mrs %0, sctlr_el2\n" + "orr %0, %0, %1\n" + "msr sctlr_el2, %0" + : "=&r" (tmp) : "L" (SCTLR_ELx_DSSBS)); +} diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 68755fd70dcf..69ff9887f724 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -12,7 +12,7 @@ lib-y := clear_user.o delay.o copy_from_user.o \ # when supported by the CPU. Result and argument registers are handled # correctly, based on the function prototype. lib-$(CONFIG_ARM64_LSE_ATOMICS) += atomic_ll_sc.o -CFLAGS_atomic_ll_sc.o := -fcall-used-x0 -ffixed-x1 -ffixed-x2 \ +CFLAGS_atomic_ll_sc.o := -ffixed-x1 -ffixed-x2 \ -ffixed-x3 -ffixed-x4 -ffixed-x5 -ffixed-x6 \ -ffixed-x7 -fcall-saved-x8 -fcall-saved-x9 \ -fcall-saved-x10 -fcall-saved-x11 -fcall-saved-x12 \ @@ -25,3 +25,5 @@ KCOV_INSTRUMENT_atomic_ll_sc.o := n UBSAN_SANITIZE_atomic_ll_sc.o := n lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o + +obj-$(CONFIG_CRC32) += crc32.o diff --git a/arch/arm64/lib/crc32.S b/arch/arm64/lib/crc32.S new file mode 100644 index 000000000000..5bc1e85b4e1c --- /dev/null +++ b/arch/arm64/lib/crc32.S @@ -0,0 +1,60 @@ +/* + * Accelerated CRC32(C) using AArch64 CRC instructions + * + * Copyright (C) 2016 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/linkage.h> +#include <asm/alternative.h> +#include <asm/assembler.h> + + .cpu generic+crc + + .macro __crc32, c +0: subs x2, x2, #16 + b.mi 8f + ldp x3, x4, [x1], #16 +CPU_BE( rev x3, x3 ) +CPU_BE( rev x4, x4 ) + crc32\c\()x w0, w0, x3 + crc32\c\()x w0, w0, x4 + b.ne 0b + ret + +8: tbz x2, #3, 4f + ldr x3, [x1], #8 +CPU_BE( rev x3, x3 ) + crc32\c\()x w0, w0, x3 +4: tbz x2, #2, 2f + ldr w3, [x1], #4 +CPU_BE( rev w3, w3 ) + crc32\c\()w w0, w0, w3 +2: tbz x2, #1, 1f + ldrh w3, [x1], #2 +CPU_BE( rev16 w3, w3 ) + crc32\c\()h w0, w0, w3 +1: tbz x2, #0, 0f + ldrb w3, [x1] + crc32\c\()b w0, w0, w3 +0: ret + .endm + + .align 5 +ENTRY(crc32_le) +alternative_if_not ARM64_HAS_CRC32 + b crc32_le_base +alternative_else_nop_endif + __crc32 +ENDPROC(crc32_le) + + .align 5 +ENTRY(__crc32c_le) +alternative_if_not ARM64_HAS_CRC32 + b __crc32c_le_base +alternative_else_nop_endif + __crc32 c +ENDPROC(__crc32c_le) diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index c127f94da8e2..1f0ea2facf24 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -88,7 +88,7 @@ void verify_cpu_asid_bits(void) } } -static void flush_context(unsigned int cpu) +static void flush_context(void) { int i; u64 asid; @@ -142,7 +142,7 @@ static bool check_update_reserved_asid(u64 asid, u64 newasid) return hit; } -static u64 new_context(struct mm_struct *mm, unsigned int cpu) +static u64 new_context(struct mm_struct *mm) { static u32 cur_idx = 1; u64 asid = atomic64_read(&mm->context.id); @@ -180,7 +180,7 @@ static u64 new_context(struct mm_struct *mm, unsigned int cpu) /* We're out of ASIDs, so increment the global generation count */ generation = atomic64_add_return_relaxed(ASID_FIRST_VERSION, &asid_generation); - flush_context(cpu); + flush_context(); /* We have more ASIDs than CPUs, so this will always succeed */ asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1); @@ -196,6 +196,9 @@ void check_and_switch_context(struct mm_struct *mm, unsigned int cpu) unsigned long flags; u64 asid, old_active_asid; + if (system_supports_cnp()) + cpu_set_reserved_ttbr0(); + asid = atomic64_read(&mm->context.id); /* @@ -223,7 +226,7 @@ void check_and_switch_context(struct mm_struct *mm, unsigned int cpu) /* Check that our ASID belongs to the current generation. */ asid = atomic64_read(&mm->context.id); if ((asid ^ atomic64_read(&asid_generation)) >> asid_bits) { - asid = new_context(mm, cpu); + asid = new_context(mm); atomic64_set(&mm->context.id, asid); } diff --git a/arch/arm64/mm/dump.c b/arch/arm64/mm/dump.c index 65dfc8571bf8..fcb1f2a6d7c6 100644 --- a/arch/arm64/mm/dump.c +++ b/arch/arm64/mm/dump.c @@ -36,8 +36,8 @@ static const struct addr_marker address_markers[] = { #endif { MODULES_VADDR, "Modules start" }, { MODULES_END, "Modules end" }, - { VMALLOC_START, "vmalloc() Area" }, - { VMALLOC_END, "vmalloc() End" }, + { VMALLOC_START, "vmalloc() area" }, + { VMALLOC_END, "vmalloc() end" }, { FIXADDR_START, "Fixmap start" }, { FIXADDR_TOP, "Fixmap end" }, { PCI_IO_START, "PCI I/O start" }, @@ -46,7 +46,7 @@ static const struct addr_marker address_markers[] = { { VMEMMAP_START, "vmemmap start" }, { VMEMMAP_START + VMEMMAP_SIZE, "vmemmap end" }, #endif - { PAGE_OFFSET, "Linear Mapping" }, + { PAGE_OFFSET, "Linear mapping" }, { -1, NULL }, }; diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 50b30ff30de4..d0e638ef3af6 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -37,6 +37,7 @@ #include <asm/cmpxchg.h> #include <asm/cpufeature.h> #include <asm/exception.h> +#include <asm/daifflags.h> #include <asm/debug-monitors.h> #include <asm/esr.h> #include <asm/sysreg.h> @@ -56,10 +57,16 @@ struct fault_info { }; static const struct fault_info fault_info[]; +static struct fault_info debug_fault_info[]; static inline const struct fault_info *esr_to_fault_info(unsigned int esr) { - return fault_info + (esr & 63); + return fault_info + (esr & ESR_ELx_FSC); +} + +static inline const struct fault_info *esr_to_debug_fault_info(unsigned int esr) +{ + return debug_fault_info + DBG_ESR_EVT(esr); } #ifdef CONFIG_KPROBES @@ -235,9 +242,8 @@ static bool is_el1_instruction_abort(unsigned int esr) return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR; } -static inline bool is_el1_permission_fault(unsigned int esr, - struct pt_regs *regs, - unsigned long addr) +static inline bool is_el1_permission_fault(unsigned long addr, unsigned int esr, + struct pt_regs *regs) { unsigned int ec = ESR_ELx_EC(esr); unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE; @@ -283,7 +289,7 @@ static void __do_kernel_fault(unsigned long addr, unsigned int esr, if (!is_el1_instruction_abort(esr) && fixup_exception(regs)) return; - if (is_el1_permission_fault(esr, regs, addr)) { + if (is_el1_permission_fault(addr, esr, regs)) { if (esr & ESR_ELx_WNR) msg = "write to read-only memory"; else @@ -454,7 +460,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, mm_flags |= FAULT_FLAG_WRITE; } - if (addr < TASK_SIZE && is_el1_permission_fault(esr, regs, addr)) { + if (addr < TASK_SIZE && is_el1_permission_fault(addr, esr, regs)) { /* regs->orig_addr_limit may be 0 if we entered from EL0 */ if (regs->orig_addr_limit == KERNEL_DS) die_kernel_fault("access to user memory with fs=KERNEL_DS", @@ -771,7 +777,7 @@ asmlinkage void __exception do_el0_ia_bp_hardening(unsigned long addr, if (addr > TASK_SIZE) arm64_apply_bp_hardening(); - local_irq_enable(); + local_daif_restore(DAIF_PROCCTX); do_mem_abort(addr, esr, regs); } @@ -785,7 +791,7 @@ asmlinkage void __exception do_sp_pc_abort(unsigned long addr, if (user_mode(regs)) { if (instruction_pointer(regs) > TASK_SIZE) arm64_apply_bp_hardening(); - local_irq_enable(); + local_daif_restore(DAIF_PROCCTX); } clear_siginfo(&info); @@ -831,7 +837,7 @@ asmlinkage int __exception do_debug_exception(unsigned long addr, unsigned int esr, struct pt_regs *regs) { - const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr); + const struct fault_info *inf = esr_to_debug_fault_info(esr); int rv; /* @@ -864,17 +870,3 @@ asmlinkage int __exception do_debug_exception(unsigned long addr, return rv; } NOKPROBE_SYMBOL(do_debug_exception); - -#ifdef CONFIG_ARM64_PAN -void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) -{ - /* - * We modify PSTATE. This won't work from irq context as the PSTATE - * is discarded once we return from the exception. - */ - WARN_ON_ONCE(in_interrupt()); - - sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0); - asm(SET_PSTATE_PAN(1)); -} -#endif /* CONFIG_ARM64_PAN */ diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 787e27964ab9..3cf87341859f 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -284,7 +284,6 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max) #endif /* CONFIG_NUMA */ -#ifdef CONFIG_HAVE_ARCH_PFN_VALID int pfn_valid(unsigned long pfn) { phys_addr_t addr = pfn << PAGE_SHIFT; @@ -294,7 +293,6 @@ int pfn_valid(unsigned long pfn) return memblock_is_map_memory(addr); } EXPORT_SYMBOL(pfn_valid); -#endif #ifndef CONFIG_SPARSEMEM static void __init arm64_memory_present(void) diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c index 12145874c02b..fccb1a6f8c6f 100644 --- a/arch/arm64/mm/kasan_init.c +++ b/arch/arm64/mm/kasan_init.c @@ -192,7 +192,7 @@ void __init kasan_init(void) /* * We are going to perform proper setup of shadow memory. - * At first we should unmap early shadow (clear_pgds() call bellow). + * At first we should unmap early shadow (clear_pgds() call below). * However, instrumented code couldn't execute without shadow memory. * tmp_pg_dir used to keep early shadow mapped until full shadow * setup will be finished. diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 8080c9f489c3..9498c15b847b 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -67,6 +67,24 @@ static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; +static DEFINE_SPINLOCK(swapper_pgdir_lock); + +void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) +{ + pgd_t *fixmap_pgdp; + + spin_lock(&swapper_pgdir_lock); + fixmap_pgdp = pgd_set_fixmap(__pa_symbol(pgdp)); + WRITE_ONCE(*fixmap_pgdp, pgd); + /* + * We need dsb(ishst) here to ensure the page-table-walker sees + * our new entry before set_p?d() returns. The fixmap's + * flush_tlb_kernel_range() via clear_fixmap() does this for us. + */ + pgd_clear_fixmap(); + spin_unlock(&swapper_pgdir_lock); +} + pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, unsigned long size, pgprot_t vma_prot) { @@ -629,34 +647,18 @@ static void __init map_kernel(pgd_t *pgdp) */ void __init paging_init(void) { - phys_addr_t pgd_phys = early_pgtable_alloc(); - pgd_t *pgdp = pgd_set_fixmap(pgd_phys); + pgd_t *pgdp = pgd_set_fixmap(__pa_symbol(swapper_pg_dir)); map_kernel(pgdp); map_mem(pgdp); - /* - * We want to reuse the original swapper_pg_dir so we don't have to - * communicate the new address to non-coherent secondaries in - * secondary_entry, and so cpu_switch_mm can generate the address with - * adrp+add rather than a load from some global variable. - * - * To do this we need to go via a temporary pgd. - */ - cpu_replace_ttbr1(__va(pgd_phys)); - memcpy(swapper_pg_dir, pgdp, PGD_SIZE); - cpu_replace_ttbr1(lm_alias(swapper_pg_dir)); - pgd_clear_fixmap(); - memblock_free(pgd_phys, PAGE_SIZE); - /* - * We only reuse the PGD from the swapper_pg_dir, not the pud + pmd - * allocated with it. - */ - memblock_free(__pa_symbol(swapper_pg_dir) + PAGE_SIZE, - __pa_symbol(swapper_pg_end) - __pa_symbol(swapper_pg_dir) - - PAGE_SIZE); + cpu_replace_ttbr1(lm_alias(swapper_pg_dir)); + init_mm.pgd = swapper_pg_dir; + + memblock_free(__pa_symbol(init_pg_dir), + __pa_symbol(init_pg_end) - __pa_symbol(init_pg_dir)); } /* diff --git a/arch/arm64/mm/numa.c b/arch/arm64/mm/numa.c index 146c04ceaa51..d7b66fc5e1c5 100644 --- a/arch/arm64/mm/numa.c +++ b/arch/arm64/mm/numa.c @@ -391,7 +391,6 @@ static int __init numa_init(int (*init_func)(void)) nodes_clear(numa_nodes_parsed); nodes_clear(node_possible_map); nodes_clear(node_online_map); - numa_free_distance(); ret = numa_alloc_distance(); if (ret < 0) @@ -399,20 +398,24 @@ static int __init numa_init(int (*init_func)(void)) ret = init_func(); if (ret < 0) - return ret; + goto out_free_distance; if (nodes_empty(numa_nodes_parsed)) { pr_info("No NUMA configuration found\n"); - return -EINVAL; + ret = -EINVAL; + goto out_free_distance; } ret = numa_register_nodes(); if (ret < 0) - return ret; + goto out_free_distance; setup_node_to_cpumask_map(); return 0; +out_free_distance: + numa_free_distance(); + return ret; } /** @@ -432,7 +435,7 @@ static int __init dummy_numa_init(void) if (numa_off) pr_info("NUMA disabled\n"); /* Forced off on command line. */ pr_info("Faking a node at [mem %#018Lx-%#018Lx]\n", - 0LLU, PFN_PHYS(max_pfn) - 1); + memblock_start_of_DRAM(), memblock_end_of_DRAM() - 1); for_each_memblock(memory, mblk) { ret = numa_add_memblk(0, mblk->base, mblk->base + mblk->size); diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 03646e6a2ef4..2c75b0b903ae 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -160,6 +160,12 @@ ENTRY(cpu_do_switch_mm) mrs x2, ttbr1_el1 mmid x1, x1 // get mm->context.id phys_to_ttbr x3, x0 + +alternative_if ARM64_HAS_CNP + cbz x1, 1f // skip CNP for reserved ASID + orr x3, x3, #TTBR_CNP_BIT +1: +alternative_else_nop_endif #ifdef CONFIG_ARM64_SW_TTBR0_PAN bfi x3, x1, #48, #16 // set the ASID field in TTBR0 #endif @@ -184,7 +190,7 @@ ENDPROC(cpu_do_switch_mm) .endm /* - * void idmap_cpu_replace_ttbr1(phys_addr_t new_pgd) + * void idmap_cpu_replace_ttbr1(phys_addr_t ttbr1) * * This is the low-level counterpart to cpu_replace_ttbr1, and should not be * called by anything else. It can only be executed from a TTBR0 mapping. @@ -194,8 +200,7 @@ ENTRY(idmap_cpu_replace_ttbr1) __idmap_cpu_set_reserved_ttbr1 x1, x3 - phys_to_ttbr x3, x0 - msr ttbr1_el1, x3 + msr ttbr1_el1, x0 isb restore_daif x2 diff --git a/arch/c6x/Kconfig b/arch/c6x/Kconfig index a641b0bf1611..f65a084607fd 100644 --- a/arch/c6x/Kconfig +++ b/arch/c6x/Kconfig @@ -9,7 +9,7 @@ config C6X select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE select CLKDEV_LOOKUP - select DMA_NONCOHERENT_OPS + select DMA_DIRECT_OPS select GENERIC_ATOMIC64 select GENERIC_IRQ_SHOW select HAVE_ARCH_TRACEHOOK diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig index 89a4b22f34d9..3ef46522e89f 100644 --- a/arch/hexagon/Kconfig +++ b/arch/hexagon/Kconfig @@ -4,6 +4,7 @@ comment "Linux Kernel Configuration for Hexagon" config HEXAGON def_bool y + select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_NO_PREEMPT select HAVE_OPROFILE # Other pending projects/to-do items. @@ -29,6 +30,7 @@ config HEXAGON select GENERIC_CLOCKEVENTS_BROADCAST select MODULES_USE_ELF_RELA select GENERIC_CPU_DEVICES + select DMA_DIRECT_OPS ---help--- Qualcomm Hexagon is a processor architecture designed for high performance and low power across a wide variety of applications. diff --git a/arch/hexagon/include/asm/Kbuild b/arch/hexagon/include/asm/Kbuild index dd2fd9c0d292..47c4da3d64a4 100644 --- a/arch/hexagon/include/asm/Kbuild +++ b/arch/hexagon/include/asm/Kbuild @@ -6,6 +6,7 @@ generic-y += compat.h generic-y += current.h generic-y += device.h generic-y += div64.h +generic-y += dma-mapping.h generic-y += emergency-restart.h generic-y += extable.h generic-y += fb.h diff --git a/arch/hexagon/include/asm/dma-mapping.h b/arch/hexagon/include/asm/dma-mapping.h deleted file mode 100644 index 263f6acbfb0f..000000000000 --- a/arch/hexagon/include/asm/dma-mapping.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * DMA operations for the Hexagon architecture - * - * Copyright (c) 2010-2011, The Linux Foundation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 and - * only version 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -#ifndef _ASM_DMA_MAPPING_H -#define _ASM_DMA_MAPPING_H - -#include <linux/types.h> -#include <linux/cache.h> -#include <linux/mm.h> -#include <linux/scatterlist.h> -#include <linux/dma-debug.h> -#include <asm/io.h> - -struct device; - -extern const struct dma_map_ops *dma_ops; - -static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) -{ - return dma_ops; -} - -#endif diff --git a/arch/hexagon/kernel/dma.c b/arch/hexagon/kernel/dma.c index 7ebe7ad19d15..706699374444 100644 --- a/arch/hexagon/kernel/dma.c +++ b/arch/hexagon/kernel/dma.c @@ -18,32 +18,19 @@ * 02110-1301, USA. */ -#include <linux/dma-mapping.h> -#include <linux/dma-direct.h> +#include <linux/dma-noncoherent.h> #include <linux/bootmem.h> #include <linux/genalloc.h> -#include <asm/dma-mapping.h> #include <linux/module.h> #include <asm/page.h> -#define HEXAGON_MAPPING_ERROR 0 - -const struct dma_map_ops *dma_ops; -EXPORT_SYMBOL(dma_ops); - -static inline void *dma_addr_to_virt(dma_addr_t dma_addr) -{ - return phys_to_virt((unsigned long) dma_addr); -} - static struct gen_pool *coherent_pool; /* Allocates from a pool of uncached memory that was reserved at boot time */ -static void *hexagon_dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_addr, gfp_t flag, - unsigned long attrs) +void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_addr, + gfp_t flag, unsigned long attrs) { void *ret; @@ -75,58 +62,17 @@ static void *hexagon_dma_alloc_coherent(struct device *dev, size_t size, return ret; } -static void hexagon_free_coherent(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_addr, unsigned long attrs) +void arch_dma_free(struct device *dev, size_t size, void *vaddr, + dma_addr_t dma_addr, unsigned long attrs) { gen_pool_free(coherent_pool, (unsigned long) vaddr, size); } -static int check_addr(const char *name, struct device *hwdev, - dma_addr_t bus, size_t size) -{ - if (hwdev && hwdev->dma_mask && !dma_capable(hwdev, bus, size)) { - if (*hwdev->dma_mask >= DMA_BIT_MASK(32)) - printk(KERN_ERR - "%s: overflow %Lx+%zu of device mask %Lx\n", - name, (long long)bus, size, - (long long)*hwdev->dma_mask); - return 0; - } - return 1; -} - -static int hexagon_map_sg(struct device *hwdev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, - unsigned long attrs) +void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, + size_t size, enum dma_data_direction dir) { - struct scatterlist *s; - int i; - - WARN_ON(nents == 0 || sg[0].length == 0); - - for_each_sg(sg, s, nents, i) { - s->dma_address = sg_phys(s); - if (!check_addr("map_sg", hwdev, s->dma_address, s->length)) - return 0; - - s->dma_length = s->length; + void *addr = phys_to_virt(paddr); - if (attrs & DMA_ATTR_SKIP_CPU_SYNC) - continue; - - flush_dcache_range(dma_addr_to_virt(s->dma_address), - dma_addr_to_virt(s->dma_address + s->length)); - } - - return nents; -} - -/* - * address is virtual - */ -static inline void dma_sync(void *addr, size_t size, - enum dma_data_direction dir) -{ switch (dir) { case DMA_TO_DEVICE: hexagon_clean_dcache_range((unsigned long) addr, @@ -144,76 +90,3 @@ static inline void dma_sync(void *addr, size_t size, BUG(); } } - -/** - * hexagon_map_page() - maps an address for device DMA - * @dev: pointer to DMA device - * @page: pointer to page struct of DMA memory - * @offset: offset within page - * @size: size of memory to map - * @dir: transfer direction - * @attrs: pointer to DMA attrs (not used) - * - * Called to map a memory address to a DMA address prior - * to accesses to/from device. - * - * We don't particularly have many hoops to jump through - * so far. Straight translation between phys and virtual. - * - * DMA is not cache coherent so sync is necessary; this - * seems to be a convenient place to do it. - * - */ -static dma_addr_t hexagon_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, - unsigned long attrs) -{ - dma_addr_t bus = page_to_phys(page) + offset; - WARN_ON(size == 0); - - if (!check_addr("map_single", dev, bus, size)) - return HEXAGON_MAPPING_ERROR; - - if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - dma_sync(dma_addr_to_virt(bus), size, dir); - - return bus; -} - -static void hexagon_sync_single_for_cpu(struct device *dev, - dma_addr_t dma_handle, size_t size, - enum dma_data_direction dir) -{ - dma_sync(dma_addr_to_virt(dma_handle), size, dir); -} - -static void hexagon_sync_single_for_device(struct device *dev, - dma_addr_t dma_handle, size_t size, - enum dma_data_direction dir) -{ - dma_sync(dma_addr_to_virt(dma_handle), size, dir); -} - -static int hexagon_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - return dma_addr == HEXAGON_MAPPING_ERROR; -} - -const struct dma_map_ops hexagon_dma_ops = { - .alloc = hexagon_dma_alloc_coherent, - .free = hexagon_free_coherent, - .map_sg = hexagon_map_sg, - .map_page = hexagon_map_page, - .sync_single_for_cpu = hexagon_sync_single_for_cpu, - .sync_single_for_device = hexagon_sync_single_for_device, - .mapping_error = hexagon_mapping_error, -}; - -void __init hexagon_dma_init(void) -{ - if (dma_ops) - return; - - dma_ops = &hexagon_dma_ops; -} diff --git a/arch/ia64/include/asm/dma-mapping.h b/arch/ia64/include/asm/dma-mapping.h index 76e4d6632d68..522745ae67bb 100644 --- a/arch/ia64/include/asm/dma-mapping.h +++ b/arch/ia64/include/asm/dma-mapping.h @@ -10,8 +10,6 @@ #include <linux/scatterlist.h> #include <linux/dma-debug.h> -#define ARCH_HAS_DMA_GET_REQUIRED_MASK - extern const struct dma_map_ops *dma_ops; extern struct ia64_machine_vector ia64_mv; extern void set_iommu_machvec(void); diff --git a/arch/ia64/include/asm/machvec.h b/arch/ia64/include/asm/machvec.h index 267f4f170191..5133739966bc 100644 --- a/arch/ia64/include/asm/machvec.h +++ b/arch/ia64/include/asm/machvec.h @@ -44,7 +44,6 @@ typedef void ia64_mv_kernel_launch_event_t(void); /* DMA-mapping interface: */ typedef void ia64_mv_dma_init (void); -typedef u64 ia64_mv_dma_get_required_mask (struct device *); typedef const struct dma_map_ops *ia64_mv_dma_get_ops(struct device *); /* @@ -127,7 +126,6 @@ extern void machvec_tlb_migrate_finish (struct mm_struct *); # define platform_global_tlb_purge ia64_mv.global_tlb_purge # define platform_tlb_migrate_finish ia64_mv.tlb_migrate_finish # define platform_dma_init ia64_mv.dma_init -# define platform_dma_get_required_mask ia64_mv.dma_get_required_mask # define platform_dma_get_ops ia64_mv.dma_get_ops # define platform_irq_to_vector ia64_mv.irq_to_vector # define platform_local_vector_to_irq ia64_mv.local_vector_to_irq @@ -171,7 +169,6 @@ struct ia64_machine_vector { ia64_mv_global_tlb_purge_t *global_tlb_purge; ia64_mv_tlb_migrate_finish_t *tlb_migrate_finish; ia64_mv_dma_init *dma_init; - ia64_mv_dma_get_required_mask *dma_get_required_mask; ia64_mv_dma_get_ops *dma_get_ops; ia64_mv_irq_to_vector *irq_to_vector; ia64_mv_local_vector_to_irq *local_vector_to_irq; @@ -211,7 +208,6 @@ struct ia64_machine_vector { platform_global_tlb_purge, \ platform_tlb_migrate_finish, \ platform_dma_init, \ - platform_dma_get_required_mask, \ platform_dma_get_ops, \ platform_irq_to_vector, \ platform_local_vector_to_irq, \ @@ -286,9 +282,6 @@ extern const struct dma_map_ops *dma_get_ops(struct device *); #ifndef platform_dma_get_ops # define platform_dma_get_ops dma_get_ops #endif -#ifndef platform_dma_get_required_mask -# define platform_dma_get_required_mask ia64_dma_get_required_mask -#endif #ifndef platform_irq_to_vector # define platform_irq_to_vector __ia64_irq_to_vector #endif diff --git a/arch/ia64/include/asm/machvec_init.h b/arch/ia64/include/asm/machvec_init.h index 2b32fd06b7c6..2aafb69a3787 100644 --- a/arch/ia64/include/asm/machvec_init.h +++ b/arch/ia64/include/asm/machvec_init.h @@ -4,7 +4,6 @@ extern ia64_mv_send_ipi_t ia64_send_ipi; extern ia64_mv_global_tlb_purge_t ia64_global_tlb_purge; -extern ia64_mv_dma_get_required_mask ia64_dma_get_required_mask; extern ia64_mv_irq_to_vector __ia64_irq_to_vector; extern ia64_mv_local_vector_to_irq __ia64_local_vector_to_irq; extern ia64_mv_pci_get_legacy_mem_t ia64_pci_get_legacy_mem; diff --git a/arch/ia64/include/asm/machvec_sn2.h b/arch/ia64/include/asm/machvec_sn2.h index ece9fa85be88..b5153d300289 100644 --- a/arch/ia64/include/asm/machvec_sn2.h +++ b/arch/ia64/include/asm/machvec_sn2.h @@ -55,7 +55,6 @@ extern ia64_mv_readb_t __sn_readb_relaxed; extern ia64_mv_readw_t __sn_readw_relaxed; extern ia64_mv_readl_t __sn_readl_relaxed; extern ia64_mv_readq_t __sn_readq_relaxed; -extern ia64_mv_dma_get_required_mask sn_dma_get_required_mask; extern ia64_mv_dma_init sn_dma_init; extern ia64_mv_migrate_t sn_migrate; extern ia64_mv_kernel_launch_event_t sn_kernel_launch_event; @@ -100,7 +99,6 @@ extern ia64_mv_pci_fixup_bus_t sn_pci_fixup_bus; #define platform_pci_get_legacy_mem sn_pci_get_legacy_mem #define platform_pci_legacy_read sn_pci_legacy_read #define platform_pci_legacy_write sn_pci_legacy_write -#define platform_dma_get_required_mask sn_dma_get_required_mask #define platform_dma_init sn_dma_init #define platform_migrate sn_migrate #define platform_kernel_launch_event sn_kernel_launch_event diff --git a/arch/ia64/pci/pci.c b/arch/ia64/pci/pci.c index 7ccc64d5fe3e..5d71800df431 100644 --- a/arch/ia64/pci/pci.c +++ b/arch/ia64/pci/pci.c @@ -568,32 +568,6 @@ static void __init set_pci_dfl_cacheline_size(void) pci_dfl_cache_line_size = (1 << cci.pcci_line_size) / 4; } -u64 ia64_dma_get_required_mask(struct device *dev) -{ - u32 low_totalram = ((max_pfn - 1) << PAGE_SHIFT); - u32 high_totalram = ((max_pfn - 1) >> (32 - PAGE_SHIFT)); - u64 mask; - - if (!high_totalram) { - /* convert to mask just covering totalram */ - low_totalram = (1 << (fls(low_totalram) - 1)); - low_totalram += low_totalram - 1; - mask = low_totalram; - } else { - high_totalram = (1 << (fls(high_totalram) - 1)); - high_totalram += high_totalram - 1; - mask = (((u64)high_totalram) << 32) + 0xffffffff; - } - return mask; -} -EXPORT_SYMBOL_GPL(ia64_dma_get_required_mask); - -u64 dma_get_required_mask(struct device *dev) -{ - return platform_dma_get_required_mask(dev); -} -EXPORT_SYMBOL_GPL(dma_get_required_mask); - static int __init pcibios_init(void) { set_pci_dfl_cacheline_size(); diff --git a/arch/ia64/sn/pci/pci_dma.c b/arch/ia64/sn/pci/pci_dma.c index 74c934a997bb..96eb2567718a 100644 --- a/arch/ia64/sn/pci/pci_dma.c +++ b/arch/ia64/sn/pci/pci_dma.c @@ -344,11 +344,10 @@ static int sn_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) return 0; } -u64 sn_dma_get_required_mask(struct device *dev) +static u64 sn_dma_get_required_mask(struct device *dev) { return DMA_BIT_MASK(64); } -EXPORT_SYMBOL_GPL(sn_dma_get_required_mask); char *sn_pci_get_legacy_mem(struct pci_bus *bus) { @@ -473,6 +472,7 @@ static struct dma_map_ops sn_dma_ops = { .sync_sg_for_device = sn_dma_sync_sg_for_device, .mapping_error = sn_dma_mapping_error, .dma_supported = sn_dma_supported, + .get_required_mask = sn_dma_get_required_mask, }; void sn_dma_init(void) diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index 070553791e97..c7b2a8d60a41 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -26,7 +26,7 @@ config M68K select MODULES_USE_ELF_RELA select OLD_SIGSUSPEND3 select OLD_SIGACTION - select DMA_NONCOHERENT_OPS if HAS_DMA + select DMA_DIRECT_OPS if HAS_DMA select HAVE_MEMBLOCK select ARCH_DISCARD_MEMBLOCK select NO_BOOTMEM diff --git a/arch/m68k/emu/nfblock.c b/arch/m68k/emu/nfblock.c index e9110b9b8bcd..38049357d6d3 100644 --- a/arch/m68k/emu/nfblock.c +++ b/arch/m68k/emu/nfblock.c @@ -73,7 +73,7 @@ static blk_qc_t nfhd_make_request(struct request_queue *queue, struct bio *bio) len = bvec.bv_len; len >>= 9; nfhd_read_write(dev->id, 0, dir, sec >> shift, len >> shift, - bvec_to_phys(&bvec)); + page_to_phys(bvec.bv_page) + bvec.bv_offset); sec += len; } bio_endio(bio); diff --git a/arch/m68k/include/asm/atafd.h b/arch/m68k/include/asm/atafd.h deleted file mode 100644 index ad7014cad633..000000000000 --- a/arch/m68k/include/asm/atafd.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_M68K_FD_H -#define _ASM_M68K_FD_H - -/* Definitions for the Atari Floppy driver */ - -struct atari_format_descr { - int track; /* to be formatted */ - int head; /* "" "" */ - int sect_offset; /* offset of first sector */ -}; - -#endif diff --git a/arch/m68k/include/asm/atafdreg.h b/arch/m68k/include/asm/atafdreg.h deleted file mode 100644 index c31b4919ed2d..000000000000 --- a/arch/m68k/include/asm/atafdreg.h +++ /dev/null @@ -1,80 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _LINUX_FDREG_H -#define _LINUX_FDREG_H - -/* -** WD1772 stuff - */ - -/* register codes */ - -#define FDCSELREG_STP (0x80) /* command/status register */ -#define FDCSELREG_TRA (0x82) /* track register */ -#define FDCSELREG_SEC (0x84) /* sector register */ -#define FDCSELREG_DTA (0x86) /* data register */ - -/* register names for FDC_READ/WRITE macros */ - -#define FDCREG_CMD 0 -#define FDCREG_STATUS 0 -#define FDCREG_TRACK 2 -#define FDCREG_SECTOR 4 -#define FDCREG_DATA 6 - -/* command opcodes */ - -#define FDCCMD_RESTORE (0x00) /* - */ -#define FDCCMD_SEEK (0x10) /* | */ -#define FDCCMD_STEP (0x20) /* | TYP 1 Commands */ -#define FDCCMD_STIN (0x40) /* | */ -#define FDCCMD_STOT (0x60) /* - */ -#define FDCCMD_RDSEC (0x80) /* - TYP 2 Commands */ -#define FDCCMD_WRSEC (0xa0) /* - " */ -#define FDCCMD_RDADR (0xc0) /* - */ -#define FDCCMD_RDTRA (0xe0) /* | TYP 3 Commands */ -#define FDCCMD_WRTRA (0xf0) /* - */ -#define FDCCMD_FORCI (0xd0) /* - TYP 4 Command */ - -/* command modifier bits */ - -#define FDCCMDADD_SR6 (0x00) /* step rate settings */ -#define FDCCMDADD_SR12 (0x01) -#define FDCCMDADD_SR2 (0x02) -#define FDCCMDADD_SR3 (0x03) -#define FDCCMDADD_V (0x04) /* verify */ -#define FDCCMDADD_H (0x08) /* wait for spin-up */ -#define FDCCMDADD_U (0x10) /* update track register */ -#define FDCCMDADD_M (0x10) /* multiple sector access */ -#define FDCCMDADD_E (0x04) /* head settling flag */ -#define FDCCMDADD_P (0x02) /* precompensation off */ -#define FDCCMDADD_A0 (0x01) /* DAM flag */ - -/* status register bits */ - -#define FDCSTAT_MOTORON (0x80) /* motor on */ -#define FDCSTAT_WPROT (0x40) /* write protected (FDCCMD_WR*) */ -#define FDCSTAT_SPINUP (0x20) /* motor speed stable (Type I) */ -#define FDCSTAT_DELDAM (0x20) /* sector has deleted DAM (Type II+III) */ -#define FDCSTAT_RECNF (0x10) /* record not found */ -#define FDCSTAT_CRC (0x08) /* CRC error */ -#define FDCSTAT_TR00 (0x04) /* Track 00 flag (Type I) */ -#define FDCSTAT_LOST (0x04) /* Lost Data (Type II+III) */ -#define FDCSTAT_IDX (0x02) /* Index status (Type I) */ -#define FDCSTAT_DRQ (0x02) /* DRQ status (Type II+III) */ -#define FDCSTAT_BUSY (0x01) /* FDC is busy */ - - -/* PSG Port A Bit Nr 0 .. Side Sel .. 0 -> Side 1 1 -> Side 2 */ -#define DSKSIDE (0x01) - -#define DSKDRVNONE (0x06) -#define DSKDRV0 (0x02) -#define DSKDRV1 (0x04) - -/* step rates */ -#define FDCSTEP_6 0x00 -#define FDCSTEP_12 0x01 -#define FDCSTEP_2 0x02 -#define FDCSTEP_3 0x03 - -#endif diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig index ace5c5bf1836..164a4857737a 100644 --- a/arch/microblaze/Kconfig +++ b/arch/microblaze/Kconfig @@ -1,6 +1,7 @@ config MICROBLAZE def_bool y select ARCH_NO_SWAP + select ARCH_HAS_DMA_COHERENT_TO_PFN if MMU select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE @@ -11,8 +12,7 @@ config MICROBLAZE select TIMER_OF select CLONE_BACKWARDS3 select COMMON_CLK - select DMA_NONCOHERENT_OPS - select DMA_NONCOHERENT_MMAP + select DMA_DIRECT_OPS select GENERIC_ATOMIC64 select GENERIC_CLOCKEVENTS select GENERIC_CPU_DEVICES diff --git a/arch/microblaze/include/asm/pgtable.h b/arch/microblaze/include/asm/pgtable.h index 7b650ab14fa0..f64ebb9c9a41 100644 --- a/arch/microblaze/include/asm/pgtable.h +++ b/arch/microblaze/include/asm/pgtable.h @@ -553,8 +553,6 @@ void __init *early_get_page(void); extern unsigned long ioremap_bot, ioremap_base; -unsigned long consistent_virt_to_pfn(void *vaddr); - void setup_memory(void); #endif /* __ASSEMBLY__ */ diff --git a/arch/microblaze/kernel/dma.c b/arch/microblaze/kernel/dma.c index 71032cf64669..a89c2d4ed5ff 100644 --- a/arch/microblaze/kernel/dma.c +++ b/arch/microblaze/kernel/dma.c @@ -42,25 +42,3 @@ void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, { __dma_sync(dev, paddr, size, dir); } - -int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t handle, size_t size, - unsigned long attrs) -{ -#ifdef CONFIG_MMU - unsigned long user_count = vma_pages(vma); - unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; - unsigned long off = vma->vm_pgoff; - unsigned long pfn; - - if (off >= count || user_count > (count - off)) - return -ENXIO; - - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - pfn = consistent_virt_to_pfn(cpu_addr); - return remap_pfn_range(vma, vma->vm_start, pfn + off, - vma->vm_end - vma->vm_start, vma->vm_page_prot); -#else - return -ENXIO; -#endif -} diff --git a/arch/microblaze/mm/consistent.c b/arch/microblaze/mm/consistent.c index c9a278ac795a..d801cc5f5b95 100644 --- a/arch/microblaze/mm/consistent.c +++ b/arch/microblaze/mm/consistent.c @@ -165,7 +165,8 @@ static pte_t *consistent_virt_to_pte(void *vaddr) return pte_offset_kernel(pmd_offset(pgd_offset_k(addr), addr), addr); } -unsigned long consistent_virt_to_pfn(void *vaddr) +long arch_dma_coherent_to_pfn(struct device *dev, void *vaddr, + dma_addr_t dma_addr) { pte_t *ptep = consistent_virt_to_pte(vaddr); diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 35511999156a..77c022e56e6e 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1106,21 +1106,22 @@ config ARCH_SUPPORTS_UPROBES bool config DMA_MAYBE_COHERENT + select ARCH_HAS_DMA_COHERENCE_H select DMA_NONCOHERENT bool config DMA_PERDEV_COHERENT bool - select DMA_MAYBE_COHERENT + select DMA_NONCOHERENT config DMA_NONCOHERENT bool + select ARCH_HAS_DMA_MMAP_PGPROT select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_HAS_SYNC_DMA_FOR_CPU select NEED_DMA_MAP_STATE - select DMA_NONCOHERENT_MMAP + select ARCH_HAS_DMA_COHERENT_TO_PFN select DMA_NONCOHERENT_CACHE_SYNC - select DMA_NONCOHERENT_OPS config SYS_HAS_EARLY_PRINTK bool diff --git a/arch/mips/alchemy/devboards/db1200.c b/arch/mips/alchemy/devboards/db1200.c index da7663770425..4bf02f96ab7f 100644 --- a/arch/mips/alchemy/devboards/db1200.c +++ b/arch/mips/alchemy/devboards/db1200.c @@ -29,8 +29,7 @@ #include <linux/leds.h> #include <linux/mmc/host.h> #include <linux/mtd/mtd.h> -#include <linux/mtd/rawnand.h> -#include <linux/mtd/partitions.h> +#include <linux/mtd/platnand.h> #include <linux/platform_device.h> #include <linux/serial_8250.h> #include <linux/spi/spi.h> @@ -197,11 +196,10 @@ static struct i2c_board_info db1200_i2c_devs[] __initdata = { /**********************************************************************/ -static void au1200_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, +static void au1200_nand_cmd_ctrl(struct nand_chip *this, int cmd, unsigned int ctrl) { - struct nand_chip *this = mtd_to_nand(mtd); - unsigned long ioaddr = (unsigned long)this->IO_ADDR_W; + unsigned long ioaddr = (unsigned long)this->legacy.IO_ADDR_W; ioaddr &= 0xffffff00; @@ -213,14 +211,14 @@ static void au1200_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, /* assume we want to r/w real data by default */ ioaddr += MEM_STNAND_DATA; } - this->IO_ADDR_R = this->IO_ADDR_W = (void __iomem *)ioaddr; + this->legacy.IO_ADDR_R = this->legacy.IO_ADDR_W = (void __iomem *)ioaddr; if (cmd != NAND_CMD_NONE) { - __raw_writeb(cmd, this->IO_ADDR_W); + __raw_writeb(cmd, this->legacy.IO_ADDR_W); wmb(); } } -static int au1200_nand_device_ready(struct mtd_info *mtd) +static int au1200_nand_device_ready(struct nand_chip *this) { return alchemy_rdsmem(AU1000_MEM_STSTAT) & 1; } diff --git a/arch/mips/alchemy/devboards/db1300.c b/arch/mips/alchemy/devboards/db1300.c index efb318e03e0a..ad7dd8e89598 100644 --- a/arch/mips/alchemy/devboards/db1300.c +++ b/arch/mips/alchemy/devboards/db1300.c @@ -19,8 +19,7 @@ #include <linux/mmc/host.h> #include <linux/module.h> #include <linux/mtd/mtd.h> -#include <linux/mtd/rawnand.h> -#include <linux/mtd/partitions.h> +#include <linux/mtd/platnand.h> #include <linux/platform_device.h> #include <linux/smsc911x.h> #include <linux/wm97xx.h> @@ -149,11 +148,10 @@ static void __init db1300_gpio_config(void) /**********************************************************************/ -static void au1300_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, +static void au1300_nand_cmd_ctrl(struct nand_chip *this, int cmd, unsigned int ctrl) { - struct nand_chip *this = mtd_to_nand(mtd); - unsigned long ioaddr = (unsigned long)this->IO_ADDR_W; + unsigned long ioaddr = (unsigned long)this->legacy.IO_ADDR_W; ioaddr &= 0xffffff00; @@ -165,14 +163,14 @@ static void au1300_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, /* assume we want to r/w real data by default */ ioaddr += MEM_STNAND_DATA; } - this->IO_ADDR_R = this->IO_ADDR_W = (void __iomem *)ioaddr; + this->legacy.IO_ADDR_R = this->legacy.IO_ADDR_W = (void __iomem *)ioaddr; if (cmd != NAND_CMD_NONE) { - __raw_writeb(cmd, this->IO_ADDR_W); + __raw_writeb(cmd, this->legacy.IO_ADDR_W); wmb(); } } -static int au1300_nand_device_ready(struct mtd_info *mtd) +static int au1300_nand_device_ready(struct nand_chip *this) { return alchemy_rdsmem(AU1000_MEM_STSTAT) & 1; } diff --git a/arch/mips/alchemy/devboards/db1550.c b/arch/mips/alchemy/devboards/db1550.c index 7d3dfaa10231..7700ad0b93b4 100644 --- a/arch/mips/alchemy/devboards/db1550.c +++ b/arch/mips/alchemy/devboards/db1550.c @@ -13,8 +13,7 @@ #include <linux/io.h> #include <linux/interrupt.h> #include <linux/mtd/mtd.h> -#include <linux/mtd/rawnand.h> -#include <linux/mtd/partitions.h> +#include <linux/mtd/platnand.h> #include <linux/platform_device.h> #include <linux/pm.h> #include <linux/spi/spi.h> @@ -126,11 +125,10 @@ static struct i2c_board_info db1550_i2c_devs[] __initdata = { /**********************************************************************/ -static void au1550_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, +static void au1550_nand_cmd_ctrl(struct nand_chip *this, int cmd, unsigned int ctrl) { - struct nand_chip *this = mtd_to_nand(mtd); - unsigned long ioaddr = (unsigned long)this->IO_ADDR_W; + unsigned long ioaddr = (unsigned long)this->legacy.IO_ADDR_W; ioaddr &= 0xffffff00; @@ -142,14 +140,14 @@ static void au1550_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, /* assume we want to r/w real data by default */ ioaddr += MEM_STNAND_DATA; } - this->IO_ADDR_R = this->IO_ADDR_W = (void __iomem *)ioaddr; + this->legacy.IO_ADDR_R = this->legacy.IO_ADDR_W = (void __iomem *)ioaddr; if (cmd != NAND_CMD_NONE) { - __raw_writeb(cmd, this->IO_ADDR_W); + __raw_writeb(cmd, this->legacy.IO_ADDR_W); wmb(); } } -static int au1550_nand_device_ready(struct mtd_info *mtd) +static int au1550_nand_device_ready(struct nand_chip *this) { return alchemy_rdsmem(AU1000_MEM_STSTAT) & 1; } diff --git a/arch/mips/include/asm/Kbuild b/arch/mips/include/asm/Kbuild index 58351e48421e..9a81e72119da 100644 --- a/arch/mips/include/asm/Kbuild +++ b/arch/mips/include/asm/Kbuild @@ -1,6 +1,7 @@ # MIPS headers generic-(CONFIG_GENERIC_CSUM) += checksum.h generic-y += current.h +generic-y += device.h generic-y += dma-contiguous.h generic-y += emergency-restart.h generic-y += export.h diff --git a/arch/mips/include/asm/device.h b/arch/mips/include/asm/device.h deleted file mode 100644 index 6aa796f1081a..000000000000 --- a/arch/mips/include/asm/device.h +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Arch specific extensions to struct device - * - * This file is released under the GPLv2 - */ -#ifndef _ASM_MIPS_DEVICE_H -#define _ASM_MIPS_DEVICE_H - -struct dev_archdata { -#ifdef CONFIG_DMA_PERDEV_COHERENT - /* Non-zero if DMA is coherent with CPU caches */ - bool dma_coherent; -#endif -}; - -struct pdev_archdata { -}; - -#endif /* _ASM_MIPS_DEVICE_H*/ diff --git a/arch/mips/include/asm/dma-coherence.h b/arch/mips/include/asm/dma-coherence.h index 8eda48748ed5..5eaa1fcc878a 100644 --- a/arch/mips/include/asm/dma-coherence.h +++ b/arch/mips/include/asm/dma-coherence.h @@ -20,6 +20,12 @@ enum coherent_io_user_state { #elif defined(CONFIG_DMA_MAYBE_COHERENT) extern enum coherent_io_user_state coherentio; extern int hw_coherentio; + +static inline bool dev_is_dma_coherent(struct device *dev) +{ + return coherentio == IO_COHERENCE_ENABLED || + (coherentio == IO_COHERENCE_DEFAULT && hw_coherentio); +} #else #ifdef CONFIG_DMA_NONCOHERENT #define coherentio IO_COHERENCE_DISABLED diff --git a/arch/mips/include/asm/dma-mapping.h b/arch/mips/include/asm/dma-mapping.h index e81c4e97ff1a..b4c477eb46ce 100644 --- a/arch/mips/include/asm/dma-mapping.h +++ b/arch/mips/include/asm/dma-mapping.h @@ -12,8 +12,6 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) return &jazz_dma_ops; #elif defined(CONFIG_SWIOTLB) return &swiotlb_dma_ops; -#elif defined(CONFIG_DMA_NONCOHERENT_OPS) - return &dma_noncoherent_ops; #else return &dma_direct_ops; #endif @@ -25,7 +23,7 @@ static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, bool coherent) { #ifdef CONFIG_DMA_PERDEV_COHERENT - dev->archdata.dma_coherent = coherent; + dev->dma_coherent = coherent; #endif } diff --git a/arch/mips/jazz/jazzdma.c b/arch/mips/jazz/jazzdma.c index d31bc2f01208..0a0aaf39fd16 100644 --- a/arch/mips/jazz/jazzdma.c +++ b/arch/mips/jazz/jazzdma.c @@ -564,13 +564,13 @@ static void *jazz_dma_alloc(struct device *dev, size_t size, { void *ret; - ret = dma_direct_alloc(dev, size, dma_handle, gfp, attrs); + ret = dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs); if (!ret) return NULL; *dma_handle = vdma_alloc(virt_to_phys(ret), size); if (*dma_handle == VDMA_ERROR) { - dma_direct_free(dev, size, ret, *dma_handle, attrs); + dma_direct_free_pages(dev, size, ret, *dma_handle, attrs); return NULL; } @@ -587,7 +587,7 @@ static void jazz_dma_free(struct device *dev, size_t size, void *vaddr, vdma_free(dma_handle); if (!(attrs & DMA_ATTR_NON_CONSISTENT)) vaddr = (void *)CAC_ADDR((unsigned long)vaddr); - return dma_direct_free(dev, size, vaddr, dma_handle, attrs); + dma_direct_free_pages(dev, size, vaddr, dma_handle, attrs); } static dma_addr_t jazz_dma_map_page(struct device *dev, struct page *page, @@ -682,7 +682,6 @@ static int jazz_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) const struct dma_map_ops jazz_dma_ops = { .alloc = jazz_dma_alloc, .free = jazz_dma_free, - .mmap = arch_dma_mmap, .map_page = jazz_dma_map_page, .unmap_page = jazz_dma_unmap_page, .map_sg = jazz_dma_map_sg, diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index 8aaaa42f91ed..e64b9e8bb002 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -1075,7 +1075,7 @@ static int __init debugfs_mips(void) arch_initcall(debugfs_mips); #endif -#if defined(CONFIG_DMA_MAYBE_COHERENT) && !defined(CONFIG_DMA_PERDEV_COHERENT) +#ifdef CONFIG_DMA_MAYBE_COHERENT /* User defined DMA coherency from command line. */ enum coherent_io_user_state coherentio = IO_COHERENCE_DEFAULT; EXPORT_SYMBOL_GPL(coherentio); diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index a9ef057c79fe..05bd77727fb9 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -1955,22 +1955,21 @@ void r4k_cache_init(void) __flush_icache_user_range = r4k_flush_icache_user_range; __local_flush_icache_user_range = local_r4k_flush_icache_user_range; -#if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT) -# if defined(CONFIG_DMA_PERDEV_COHERENT) - if (0) { -# else - if ((coherentio == IO_COHERENCE_ENABLED) || - ((coherentio == IO_COHERENCE_DEFAULT) && hw_coherentio)) { -# endif +#ifdef CONFIG_DMA_NONCOHERENT +#ifdef CONFIG_DMA_MAYBE_COHERENT + if (coherentio == IO_COHERENCE_ENABLED || + (coherentio == IO_COHERENCE_DEFAULT && hw_coherentio)) { _dma_cache_wback_inv = (void *)cache_noop; _dma_cache_wback = (void *)cache_noop; _dma_cache_inv = (void *)cache_noop; - } else { + } else +#endif /* CONFIG_DMA_MAYBE_COHERENT */ + { _dma_cache_wback_inv = r4k_dma_cache_wback_inv; _dma_cache_wback = r4k_dma_cache_wback_inv; _dma_cache_inv = r4k_dma_cache_inv; } -#endif +#endif /* CONFIG_DMA_NONCOHERENT */ build_clear_page(); build_copy_page(); diff --git a/arch/mips/mm/dma-noncoherent.c b/arch/mips/mm/dma-noncoherent.c index 2aca1236af36..e6c9485cadcf 100644 --- a/arch/mips/mm/dma-noncoherent.c +++ b/arch/mips/mm/dma-noncoherent.c @@ -14,26 +14,6 @@ #include <asm/dma-coherence.h> #include <asm/io.h> -#ifdef CONFIG_DMA_PERDEV_COHERENT -static inline int dev_is_coherent(struct device *dev) -{ - return dev->archdata.dma_coherent; -} -#else -static inline int dev_is_coherent(struct device *dev) -{ - switch (coherentio) { - default: - case IO_COHERENCE_DEFAULT: - return hw_coherentio; - case IO_COHERENCE_ENABLED: - return 1; - case IO_COHERENCE_DISABLED: - return 0; - } -} -#endif /* CONFIG_DMA_PERDEV_COHERENT */ - /* * The affected CPUs below in 'cpu_needs_post_dma_flush()' can speculatively * fill random cachelines with stale data at any time, requiring an extra @@ -49,9 +29,6 @@ static inline int dev_is_coherent(struct device *dev) */ static inline bool cpu_needs_post_dma_flush(struct device *dev) { - if (dev_is_coherent(dev)) - return false; - switch (boot_cpu_type()) { case CPU_R10000: case CPU_R12000: @@ -72,11 +49,8 @@ void *arch_dma_alloc(struct device *dev, size_t size, { void *ret; - ret = dma_direct_alloc(dev, size, dma_handle, gfp, attrs); - if (!ret) - return NULL; - - if (!dev_is_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) { + ret = dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs); + if (!ret && !(attrs & DMA_ATTR_NON_CONSISTENT)) { dma_cache_wback_inv((unsigned long) ret, size); ret = (void *)UNCAC_ADDR(ret); } @@ -87,43 +61,24 @@ void *arch_dma_alloc(struct device *dev, size_t size, void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { - if (!(attrs & DMA_ATTR_NON_CONSISTENT) && !dev_is_coherent(dev)) + if (!(attrs & DMA_ATTR_NON_CONSISTENT)) cpu_addr = (void *)CAC_ADDR((unsigned long)cpu_addr); - dma_direct_free(dev, size, cpu_addr, dma_addr, attrs); + dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs); } -int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs) +long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr, + dma_addr_t dma_addr) { - unsigned long user_count = vma_pages(vma); - unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; - unsigned long addr = (unsigned long)cpu_addr; - unsigned long off = vma->vm_pgoff; - unsigned long pfn; - int ret = -ENXIO; - - if (!dev_is_coherent(dev)) - addr = CAC_ADDR(addr); - - pfn = page_to_pfn(virt_to_page((void *)addr)); + unsigned long addr = CAC_ADDR((unsigned long)cpu_addr); + return page_to_pfn(virt_to_page((void *)addr)); +} +pgprot_t arch_dma_mmap_pgprot(struct device *dev, pgprot_t prot, + unsigned long attrs) +{ if (attrs & DMA_ATTR_WRITE_COMBINE) - vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); - else - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - - if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) - return ret; - - if (off < count && user_count <= (count - off)) { - ret = remap_pfn_range(vma, vma->vm_start, - pfn + off, - user_count << PAGE_SHIFT, - vma->vm_page_prot); - } - - return ret; + return pgprot_writecombine(prot); + return pgprot_noncached(prot); } static inline void dma_sync_virt(void *addr, size_t size, @@ -187,8 +142,7 @@ static inline void dma_sync_phys(phys_addr_t paddr, size_t size, void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, size_t size, enum dma_data_direction dir) { - if (!dev_is_coherent(dev)) - dma_sync_phys(paddr, size, dir); + dma_sync_phys(paddr, size, dir); } void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, @@ -203,6 +157,5 @@ void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size, { BUG_ON(direction == DMA_NONE); - if (!dev_is_coherent(dev)) - dma_sync_virt(vaddr, size, direction); + dma_sync_virt(vaddr, size, direction); } diff --git a/arch/mips/netlogic/xlr/platform-flash.c b/arch/mips/netlogic/xlr/platform-flash.c index 4d1b4c003376..cf9162284b07 100644 --- a/arch/mips/netlogic/xlr/platform-flash.c +++ b/arch/mips/netlogic/xlr/platform-flash.c @@ -19,8 +19,7 @@ #include <linux/mtd/mtd.h> #include <linux/mtd/physmap.h> -#include <linux/mtd/rawnand.h> -#include <linux/mtd/partitions.h> +#include <linux/mtd/platnand.h> #include <asm/netlogic/haldefs.h> #include <asm/netlogic/xlr/iomap.h> @@ -92,8 +91,8 @@ struct xlr_nand_flash_priv { static struct xlr_nand_flash_priv nand_priv; -static void xlr_nand_ctrl(struct mtd_info *mtd, int cmd, - unsigned int ctrl) +static void xlr_nand_ctrl(struct nand_chip *chip, int cmd, + unsigned int ctrl) { if (ctrl & NAND_CLE) nlm_write_reg(nand_priv.flash_mmio, diff --git a/arch/mips/pnx833x/common/platform.c b/arch/mips/pnx833x/common/platform.c index a7a4e9f5146d..dafbf027fad0 100644 --- a/arch/mips/pnx833x/common/platform.c +++ b/arch/mips/pnx833x/common/platform.c @@ -30,8 +30,7 @@ #include <linux/resource.h> #include <linux/serial.h> #include <linux/serial_pnx8xxx.h> -#include <linux/mtd/rawnand.h> -#include <linux/mtd/partitions.h> +#include <linux/mtd/platnand.h> #include <irq.h> #include <irq-mapping.h> @@ -178,10 +177,9 @@ static struct platform_device pnx833x_sata_device = { }; static void -pnx833x_flash_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl) +pnx833x_flash_nand_cmd_ctrl(struct nand_chip *this, int cmd, unsigned int ctrl) { - struct nand_chip *this = mtd_to_nand(mtd); - unsigned long nandaddr = (unsigned long)this->IO_ADDR_W; + unsigned long nandaddr = (unsigned long)this->legacy.IO_ADDR_W; if (cmd == NAND_CMD_NONE) return; diff --git a/arch/mips/rb532/devices.c b/arch/mips/rb532/devices.c index 354d258396ff..2b23ad640f39 100644 --- a/arch/mips/rb532/devices.c +++ b/arch/mips/rb532/devices.c @@ -20,9 +20,8 @@ #include <linux/ctype.h> #include <linux/string.h> #include <linux/platform_device.h> -#include <linux/mtd/rawnand.h> +#include <linux/mtd/platnand.h> #include <linux/mtd/mtd.h> -#include <linux/mtd/partitions.h> #include <linux/gpio.h> #include <linux/gpio_keys.h> #include <linux/input.h> @@ -141,14 +140,13 @@ static struct platform_device cf_slot0 = { }; /* Resources and device for NAND */ -static int rb532_dev_ready(struct mtd_info *mtd) +static int rb532_dev_ready(struct nand_chip *chip) { return gpio_get_value(GPIO_RDY); } -static void rb532_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl) +static void rb532_cmd_ctrl(struct nand_chip *chip, int cmd, unsigned int ctrl) { - struct nand_chip *chip = mtd_to_nand(mtd); unsigned char orbits, nandbits; if (ctrl & NAND_CTRL_CHANGE) { @@ -161,7 +159,7 @@ static void rb532_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl) set_latch_u5(orbits, nandbits); } if (cmd != NAND_CMD_NONE) - writeb(cmd, chip->IO_ADDR_W); + writeb(cmd, chip->legacy.IO_ADDR_W); } static struct resource nand_slot0_res[] = { diff --git a/arch/nds32/Kconfig b/arch/nds32/Kconfig index 7068f341133d..56992330026a 100644 --- a/arch/nds32/Kconfig +++ b/arch/nds32/Kconfig @@ -11,7 +11,7 @@ config NDS32 select CLKSRC_MMIO select CLONE_BACKWARDS select COMMON_CLK - select DMA_NONCOHERENT_OPS + select DMA_DIRECT_OPS select GENERIC_ATOMIC64 select GENERIC_CPU_DEVICES select GENERIC_CLOCKEVENTS diff --git a/arch/nios2/Kconfig b/arch/nios2/Kconfig index f4ad1138e6b9..03965692fbfe 100644 --- a/arch/nios2/Kconfig +++ b/arch/nios2/Kconfig @@ -4,7 +4,7 @@ config NIOS2 select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_NO_SWAP - select DMA_NONCOHERENT_OPS + select DMA_DIRECT_OPS select TIMER_OF select GENERIC_ATOMIC64 select GENERIC_CLOCKEVENTS diff --git a/arch/openrisc/Kconfig b/arch/openrisc/Kconfig index e0081e734827..a655ae280637 100644 --- a/arch/openrisc/Kconfig +++ b/arch/openrisc/Kconfig @@ -7,7 +7,7 @@ config OPENRISC def_bool y select ARCH_HAS_SYNC_DMA_FOR_DEVICE - select DMA_NONCOHERENT_OPS + select DMA_DIRECT_OPS select OF select OF_EARLY_FLATTREE select IRQ_DOMAIN diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 8e6d83f79e72..f1cd12afd943 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -186,7 +186,7 @@ config PA11 depends on PA7000 || PA7100LC || PA7200 || PA7300LC select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE - select DMA_NONCOHERENT_OPS + select DMA_DIRECT_OPS select DMA_NONCOHERENT_CACHE_SYNC config PREFETCH diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c index 4e87c35c22b7..755e89ec828a 100644 --- a/arch/parisc/kernel/setup.c +++ b/arch/parisc/kernel/setup.c @@ -102,7 +102,7 @@ void __init dma_ops_init(void) case pcxl: /* falls through */ case pcxs: case pcxt: - hppa_dma_ops = &dma_noncoherent_ops; + hppa_dma_ops = &dma_direct_ops; break; default: break; diff --git a/arch/parisc/kernel/unwind.c b/arch/parisc/kernel/unwind.c index f329b466e68f..2d14f17838d2 100644 --- a/arch/parisc/kernel/unwind.c +++ b/arch/parisc/kernel/unwind.c @@ -426,7 +426,7 @@ void unwind_frame_init_task(struct unwind_frame_info *info, r.gr[30] = get_parisc_stackpointer(); regs = &r; } - unwind_frame_init(info, task, &r); + unwind_frame_init(info, task, regs); } else { unwind_frame_init_from_blocked_task(info, task); } diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index 1fb7b6d72baf..475d786a65b0 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -7,6 +7,7 @@ config SUPERH select ARCH_NO_COHERENT_DMA_MMAP if !MMU select HAVE_PATA_PLATFORM select CLKDEV_LOOKUP + select DMA_DIRECT_OPS select HAVE_IDE if HAS_IOPORT_MAP select HAVE_MEMBLOCK select HAVE_MEMBLOCK_NODE_MAP @@ -158,13 +159,11 @@ config SWAP_IO_SPACE bool config DMA_COHERENT - select DMA_DIRECT_OPS bool config DMA_NONCOHERENT def_bool !DMA_COHERENT select ARCH_HAS_SYNC_DMA_FOR_DEVICE - select DMA_NONCOHERENT_OPS config PGTABLE_LEVELS default 3 if X2TLB diff --git a/arch/sh/boards/mach-migor/setup.c b/arch/sh/boards/mach-migor/setup.c index 254f2c662703..f4ad33c6d2aa 100644 --- a/arch/sh/boards/mach-migor/setup.c +++ b/arch/sh/boards/mach-migor/setup.c @@ -14,7 +14,7 @@ #include <linux/mmc/host.h> #include <linux/mtd/physmap.h> #include <linux/mfd/tmio.h> -#include <linux/mtd/rawnand.h> +#include <linux/mtd/platnand.h> #include <linux/i2c.h> #include <linux/regulator/fixed.h> #include <linux/regulator/machine.h> @@ -165,23 +165,21 @@ static struct mtd_partition migor_nand_flash_partitions[] = { }, }; -static void migor_nand_flash_cmd_ctl(struct mtd_info *mtd, int cmd, +static void migor_nand_flash_cmd_ctl(struct nand_chip *chip, int cmd, unsigned int ctrl) { - struct nand_chip *chip = mtd_to_nand(mtd); - if (cmd == NAND_CMD_NONE) return; if (ctrl & NAND_CLE) - writeb(cmd, chip->IO_ADDR_W + 0x00400000); + writeb(cmd, chip->legacy.IO_ADDR_W + 0x00400000); else if (ctrl & NAND_ALE) - writeb(cmd, chip->IO_ADDR_W + 0x00800000); + writeb(cmd, chip->legacy.IO_ADDR_W + 0x00800000); else - writeb(cmd, chip->IO_ADDR_W); + writeb(cmd, chip->legacy.IO_ADDR_W); } -static int migor_nand_flash_ready(struct mtd_info *mtd) +static int migor_nand_flash_ready(struct nand_chip *chip) { return gpio_get_value(GPIO_PTA1); /* NAND_RBn */ } diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index e6f2a38d2e61..7e2aa59fcc29 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -51,7 +51,7 @@ config SPARC config SPARC32 def_bool !64BIT select ARCH_HAS_SYNC_DMA_FOR_CPU - select DMA_NONCOHERENT_OPS + select DMA_DIRECT_OPS select GENERIC_ATOMIC64 select CLZ_TAB select HAVE_UID16 diff --git a/arch/sparc/include/asm/cpudata_64.h b/arch/sparc/include/asm/cpudata_64.h index 666d6b5c0440..9c3fc03abe9a 100644 --- a/arch/sparc/include/asm/cpudata_64.h +++ b/arch/sparc/include/asm/cpudata_64.h @@ -28,7 +28,7 @@ typedef struct { unsigned short sock_id; /* physical package */ unsigned short core_id; unsigned short max_cache_id; /* groupings of highest shared cache */ - unsigned short proc_id; /* strand (aka HW thread) id */ + signed short proc_id; /* strand (aka HW thread) id */ } cpuinfo_sparc; DECLARE_PER_CPU(cpuinfo_sparc, __cpu_data); diff --git a/arch/sparc/include/asm/dma-mapping.h b/arch/sparc/include/asm/dma-mapping.h index e17566376934..b0bb2fcaf1c9 100644 --- a/arch/sparc/include/asm/dma-mapping.h +++ b/arch/sparc/include/asm/dma-mapping.h @@ -14,11 +14,11 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) { #ifdef CONFIG_SPARC_LEON if (sparc_cpu_model == sparc_leon) - return &dma_noncoherent_ops; + return &dma_direct_ops; #endif #if defined(CONFIG_SPARC32) && defined(CONFIG_PCI) if (bus == &pci_bus_type) - return &dma_noncoherent_ops; + return &dma_direct_ops; #endif return dma_ops; } diff --git a/arch/sparc/include/uapi/asm/unistd.h b/arch/sparc/include/uapi/asm/unistd.h index 09acf0ddec10..45b4bf1875e6 100644 --- a/arch/sparc/include/uapi/asm/unistd.h +++ b/arch/sparc/include/uapi/asm/unistd.h @@ -427,8 +427,9 @@ #define __NR_preadv2 358 #define __NR_pwritev2 359 #define __NR_statx 360 +#define __NR_io_pgetevents 361 -#define NR_syscalls 361 +#define NR_syscalls 362 /* Bitmask values returned from kern_features system call. */ #define KERN_FEATURE_MIXED_MODE_STACK 0x00000001 diff --git a/arch/sparc/kernel/auxio_64.c b/arch/sparc/kernel/auxio_64.c index cc42225c20f3..4e8f56c3793c 100644 --- a/arch/sparc/kernel/auxio_64.c +++ b/arch/sparc/kernel/auxio_64.c @@ -115,8 +115,8 @@ static int auxio_probe(struct platform_device *dev) auxio_devtype = AUXIO_TYPE_SBUS; size = 1; } else { - printk("auxio: Unknown parent bus type [%pOFn]\n", - dp->parent); + printk("auxio: Unknown parent bus type [%s]\n", + dp->parent->name); return -ENODEV; } auxio_register = of_ioremap(&dev->resource[0], 0, size, "auxio"); diff --git a/arch/sparc/kernel/perf_event.c b/arch/sparc/kernel/perf_event.c index d3149baaa33c..67b3e6b3ce5d 100644 --- a/arch/sparc/kernel/perf_event.c +++ b/arch/sparc/kernel/perf_event.c @@ -24,6 +24,7 @@ #include <asm/cpudata.h> #include <linux/uaccess.h> #include <linux/atomic.h> +#include <linux/sched/clock.h> #include <asm/nmi.h> #include <asm/pcr.h> #include <asm/cacheflush.h> @@ -927,6 +928,8 @@ static void read_in_all_counters(struct cpu_hw_events *cpuc) sparc_perf_event_update(cp, &cp->hw, cpuc->current_idx[i]); cpuc->current_idx[i] = PIC_NO_INDEX; + if (cp->hw.state & PERF_HES_STOPPED) + cp->hw.state |= PERF_HES_ARCH; } } } @@ -959,10 +962,12 @@ static void calculate_single_pcr(struct cpu_hw_events *cpuc) enc = perf_event_get_enc(cpuc->events[i]); cpuc->pcr[0] &= ~mask_for_index(idx); - if (hwc->state & PERF_HES_STOPPED) + if (hwc->state & PERF_HES_ARCH) { cpuc->pcr[0] |= nop_for_index(idx); - else + } else { cpuc->pcr[0] |= event_encoding(enc, idx); + hwc->state = 0; + } } out: cpuc->pcr[0] |= cpuc->event[0]->hw.config_base; @@ -988,6 +993,9 @@ static void calculate_multiple_pcrs(struct cpu_hw_events *cpuc) cpuc->current_idx[i] = idx; + if (cp->hw.state & PERF_HES_ARCH) + continue; + sparc_pmu_start(cp, PERF_EF_RELOAD); } out: @@ -1079,6 +1087,8 @@ static void sparc_pmu_start(struct perf_event *event, int flags) event->hw.state = 0; sparc_pmu_enable_event(cpuc, &event->hw, idx); + + perf_event_update_userpage(event); } static void sparc_pmu_stop(struct perf_event *event, int flags) @@ -1371,9 +1381,9 @@ static int sparc_pmu_add(struct perf_event *event, int ef_flags) cpuc->events[n0] = event->hw.event_base; cpuc->current_idx[n0] = PIC_NO_INDEX; - event->hw.state = PERF_HES_UPTODATE; + event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; if (!(ef_flags & PERF_EF_START)) - event->hw.state |= PERF_HES_STOPPED; + event->hw.state |= PERF_HES_ARCH; /* * If group events scheduling transaction was started, @@ -1603,6 +1613,8 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self, struct perf_sample_data data; struct cpu_hw_events *cpuc; struct pt_regs *regs; + u64 finish_clock; + u64 start_clock; int i; if (!atomic_read(&active_events)) @@ -1616,6 +1628,8 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self, return NOTIFY_DONE; } + start_clock = sched_clock(); + regs = args->regs; cpuc = this_cpu_ptr(&cpu_hw_events); @@ -1654,6 +1668,10 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self, sparc_pmu_stop(event, 0); } + finish_clock = sched_clock(); + + perf_sample_event_took(finish_clock - start_clock); + return NOTIFY_STOP; } diff --git a/arch/sparc/kernel/power.c b/arch/sparc/kernel/power.c index d941875dd718..92627abce311 100644 --- a/arch/sparc/kernel/power.c +++ b/arch/sparc/kernel/power.c @@ -41,8 +41,8 @@ static int power_probe(struct platform_device *op) power_reg = of_ioremap(res, 0, 0x4, "power"); - printk(KERN_INFO "%pOFn: Control reg at %llx\n", - op->dev.of_node, res->start); + printk(KERN_INFO "%s: Control reg at %llx\n", + op->dev.of_node->name, res->start); if (has_button_interrupt(irq, op->dev.of_node)) { if (request_irq(irq, diff --git a/arch/sparc/kernel/prom_32.c b/arch/sparc/kernel/prom_32.c index 17c87d29ff20..b51cbb9e87dc 100644 --- a/arch/sparc/kernel/prom_32.c +++ b/arch/sparc/kernel/prom_32.c @@ -68,8 +68,8 @@ static void __init sparc32_path_component(struct device_node *dp, char *tmp_buf) return; regs = rprop->value; - sprintf(tmp_buf, "%pOFn@%x,%x", - dp, + sprintf(tmp_buf, "%s@%x,%x", + dp->name, regs->which_io, regs->phys_addr); } @@ -84,8 +84,8 @@ static void __init sbus_path_component(struct device_node *dp, char *tmp_buf) return; regs = prop->value; - sprintf(tmp_buf, "%pOFn@%x,%x", - dp, + sprintf(tmp_buf, "%s@%x,%x", + dp->name, regs->which_io, regs->phys_addr); } @@ -104,13 +104,13 @@ static void __init pci_path_component(struct device_node *dp, char *tmp_buf) regs = prop->value; devfn = (regs->phys_hi >> 8) & 0xff; if (devfn & 0x07) { - sprintf(tmp_buf, "%pOFn@%x,%x", - dp, + sprintf(tmp_buf, "%s@%x,%x", + dp->name, devfn >> 3, devfn & 0x07); } else { - sprintf(tmp_buf, "%pOFn@%x", - dp, + sprintf(tmp_buf, "%s@%x", + dp->name, devfn >> 3); } } @@ -127,8 +127,8 @@ static void __init ebus_path_component(struct device_node *dp, char *tmp_buf) regs = prop->value; - sprintf(tmp_buf, "%pOFn@%x,%x", - dp, + sprintf(tmp_buf, "%s@%x,%x", + dp->name, regs->which_io, regs->phys_addr); } @@ -167,8 +167,8 @@ static void __init ambapp_path_component(struct device_node *dp, char *tmp_buf) return; device = prop->value; - sprintf(tmp_buf, "%pOFn:%d:%d@%x,%x", - dp, *vendor, *device, + sprintf(tmp_buf, "%s:%d:%d@%x,%x", + dp->name, *vendor, *device, *intr, reg0); } @@ -201,7 +201,7 @@ char * __init build_path_component(struct device_node *dp) tmp_buf[0] = '\0'; __build_path_component(dp, tmp_buf); if (tmp_buf[0] == '\0') - snprintf(tmp_buf, sizeof(tmp_buf), "%pOFn", dp); + strcpy(tmp_buf, dp->name); n = prom_early_alloc(strlen(tmp_buf) + 1); strcpy(n, tmp_buf); diff --git a/arch/sparc/kernel/prom_64.c b/arch/sparc/kernel/prom_64.c index 6220411ce8fc..baeaeed64993 100644 --- a/arch/sparc/kernel/prom_64.c +++ b/arch/sparc/kernel/prom_64.c @@ -82,8 +82,8 @@ static void __init sun4v_path_component(struct device_node *dp, char *tmp_buf) regs = rprop->value; if (!of_node_is_root(dp->parent)) { - sprintf(tmp_buf, "%pOFn@%x,%x", - dp, + sprintf(tmp_buf, "%s@%x,%x", + dp->name, (unsigned int) (regs->phys_addr >> 32UL), (unsigned int) (regs->phys_addr & 0xffffffffUL)); return; @@ -97,17 +97,17 @@ static void __init sun4v_path_component(struct device_node *dp, char *tmp_buf) const char *prefix = (type == 0) ? "m" : "i"; if (low_bits) - sprintf(tmp_buf, "%pOFn@%s%x,%x", - dp, prefix, + sprintf(tmp_buf, "%s@%s%x,%x", + dp->name, prefix, high_bits, low_bits); else - sprintf(tmp_buf, "%pOFn@%s%x", - dp, + sprintf(tmp_buf, "%s@%s%x", + dp->name, prefix, high_bits); } else if (type == 12) { - sprintf(tmp_buf, "%pOFn@%x", - dp, high_bits); + sprintf(tmp_buf, "%s@%x", + dp->name, high_bits); } } @@ -122,8 +122,8 @@ static void __init sun4u_path_component(struct device_node *dp, char *tmp_buf) regs = prop->value; if (!of_node_is_root(dp->parent)) { - sprintf(tmp_buf, "%pOFn@%x,%x", - dp, + sprintf(tmp_buf, "%s@%x,%x", + dp->name, (unsigned int) (regs->phys_addr >> 32UL), (unsigned int) (regs->phys_addr & 0xffffffffUL)); return; @@ -138,8 +138,8 @@ static void __init sun4u_path_component(struct device_node *dp, char *tmp_buf) if (tlb_type >= cheetah) mask = 0x7fffff; - sprintf(tmp_buf, "%pOFn@%x,%x", - dp, + sprintf(tmp_buf, "%s@%x,%x", + dp->name, *(u32 *)prop->value, (unsigned int) (regs->phys_addr & mask)); } @@ -156,8 +156,8 @@ static void __init sbus_path_component(struct device_node *dp, char *tmp_buf) return; regs = prop->value; - sprintf(tmp_buf, "%pOFn@%x,%x", - dp, + sprintf(tmp_buf, "%s@%x,%x", + dp->name, regs->which_io, regs->phys_addr); } @@ -176,13 +176,13 @@ static void __init pci_path_component(struct device_node *dp, char *tmp_buf) regs = prop->value; devfn = (regs->phys_hi >> 8) & 0xff; if (devfn & 0x07) { - sprintf(tmp_buf, "%pOFn@%x,%x", - dp, + sprintf(tmp_buf, "%s@%x,%x", + dp->name, devfn >> 3, devfn & 0x07); } else { - sprintf(tmp_buf, "%pOFn@%x", - dp, + sprintf(tmp_buf, "%s@%x", + dp->name, devfn >> 3); } } @@ -203,8 +203,8 @@ static void __init upa_path_component(struct device_node *dp, char *tmp_buf) if (!prop) return; - sprintf(tmp_buf, "%pOFn@%x,%x", - dp, + sprintf(tmp_buf, "%s@%x,%x", + dp->name, *(u32 *) prop->value, (unsigned int) (regs->phys_addr & 0xffffffffUL)); } @@ -221,7 +221,7 @@ static void __init vdev_path_component(struct device_node *dp, char *tmp_buf) regs = prop->value; - sprintf(tmp_buf, "%pOFn@%x", dp, *regs); + sprintf(tmp_buf, "%s@%x", dp->name, *regs); } /* "name@addrhi,addrlo" */ @@ -236,8 +236,8 @@ static void __init ebus_path_component(struct device_node *dp, char *tmp_buf) regs = prop->value; - sprintf(tmp_buf, "%pOFn@%x,%x", - dp, + sprintf(tmp_buf, "%s@%x,%x", + dp->name, (unsigned int) (regs->phys_addr >> 32UL), (unsigned int) (regs->phys_addr & 0xffffffffUL)); } @@ -257,8 +257,8 @@ static void __init i2c_path_component(struct device_node *dp, char *tmp_buf) /* This actually isn't right... should look at the #address-cells * property of the i2c bus node etc. etc. */ - sprintf(tmp_buf, "%pOFn@%x,%x", - dp, regs[0], regs[1]); + sprintf(tmp_buf, "%s@%x,%x", + dp->name, regs[0], regs[1]); } /* "name@reg0[,reg1]" */ @@ -274,11 +274,11 @@ static void __init usb_path_component(struct device_node *dp, char *tmp_buf) regs = prop->value; if (prop->length == sizeof(u32) || regs[1] == 1) { - sprintf(tmp_buf, "%pOFn@%x", - dp, regs[0]); + sprintf(tmp_buf, "%s@%x", + dp->name, regs[0]); } else { - sprintf(tmp_buf, "%pOFn@%x,%x", - dp, regs[0], regs[1]); + sprintf(tmp_buf, "%s@%x,%x", + dp->name, regs[0], regs[1]); } } @@ -295,11 +295,11 @@ static void __init ieee1394_path_component(struct device_node *dp, char *tmp_buf regs = prop->value; if (regs[2] || regs[3]) { - sprintf(tmp_buf, "%pOFn@%08x%08x,%04x%08x", - dp, regs[0], regs[1], regs[2], regs[3]); + sprintf(tmp_buf, "%s@%08x%08x,%04x%08x", + dp->name, regs[0], regs[1], regs[2], regs[3]); } else { - sprintf(tmp_buf, "%pOFn@%08x%08x", - dp, regs[0], regs[1]); + sprintf(tmp_buf, "%s@%08x%08x", + dp->name, regs[0], regs[1]); } } @@ -361,7 +361,7 @@ char * __init build_path_component(struct device_node *dp) tmp_buf[0] = '\0'; __build_path_component(dp, tmp_buf); if (tmp_buf[0] == '\0') - snprintf(tmp_buf, sizeof(tmp_buf), "%pOFn", dp); + strcpy(tmp_buf, dp->name); n = prom_early_alloc(strlen(tmp_buf) + 1); strcpy(n, tmp_buf); diff --git a/arch/sparc/kernel/rtrap_64.S b/arch/sparc/kernel/rtrap_64.S index f6528884a2c8..4073e2b87dd0 100644 --- a/arch/sparc/kernel/rtrap_64.S +++ b/arch/sparc/kernel/rtrap_64.S @@ -84,8 +84,9 @@ __handle_signal: ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 sethi %hi(0xf << 20), %l4 and %l1, %l4, %l4 + andn %l1, %l4, %l1 ba,pt %xcc, __handle_preemption_continue - andn %l1, %l4, %l1 + srl %l4, 20, %l4 /* When returning from a NMI (%pil==15) interrupt we want to * avoid running softirqs, doing IRQ tracing, preempting, etc. diff --git a/arch/sparc/kernel/systbls_32.S b/arch/sparc/kernel/systbls_32.S index 12bee14b552c..621a363098ec 100644 --- a/arch/sparc/kernel/systbls_32.S +++ b/arch/sparc/kernel/systbls_32.S @@ -90,4 +90,4 @@ sys_call_table: /*345*/ .long sys_renameat2, sys_seccomp, sys_getrandom, sys_memfd_create, sys_bpf /*350*/ .long sys_execveat, sys_membarrier, sys_userfaultfd, sys_bind, sys_listen /*355*/ .long sys_setsockopt, sys_mlock2, sys_copy_file_range, sys_preadv2, sys_pwritev2 -/*360*/ .long sys_statx +/*360*/ .long sys_statx, sys_io_pgetevents diff --git a/arch/sparc/kernel/systbls_64.S b/arch/sparc/kernel/systbls_64.S index 387ef993880a..bb68c805b891 100644 --- a/arch/sparc/kernel/systbls_64.S +++ b/arch/sparc/kernel/systbls_64.S @@ -91,7 +91,7 @@ sys_call_table32: .word sys_renameat2, sys_seccomp, sys_getrandom, sys_memfd_create, sys_bpf /*350*/ .word sys32_execveat, sys_membarrier, sys_userfaultfd, sys_bind, sys_listen .word compat_sys_setsockopt, sys_mlock2, sys_copy_file_range, compat_sys_preadv2, compat_sys_pwritev2 -/*360*/ .word sys_statx +/*360*/ .word sys_statx, compat_sys_io_pgetevents #endif /* CONFIG_COMPAT */ @@ -173,4 +173,4 @@ sys_call_table: .word sys_renameat2, sys_seccomp, sys_getrandom, sys_memfd_create, sys_bpf /*350*/ .word sys64_execveat, sys_membarrier, sys_userfaultfd, sys_bind, sys_listen .word sys_setsockopt, sys_mlock2, sys_copy_file_range, sys_preadv2, sys_pwritev2 -/*360*/ .word sys_statx +/*360*/ .word sys_statx, sys_io_pgetevents diff --git a/arch/sparc/vdso/vclock_gettime.c b/arch/sparc/vdso/vclock_gettime.c index 3feb3d960ca5..75dca9aab737 100644 --- a/arch/sparc/vdso/vclock_gettime.c +++ b/arch/sparc/vdso/vclock_gettime.c @@ -33,9 +33,19 @@ #define TICK_PRIV_BIT (1ULL << 63) #endif +#ifdef CONFIG_SPARC64 #define SYSCALL_STRING \ "ta 0x6d;" \ - "sub %%g0, %%o0, %%o0;" \ + "bcs,a 1f;" \ + " sub %%g0, %%o0, %%o0;" \ + "1:" +#else +#define SYSCALL_STRING \ + "ta 0x10;" \ + "bcs,a 1f;" \ + " sub %%g0, %%o0, %%o0;" \ + "1:" +#endif #define SYSCALL_CLOBBERS \ "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", \ diff --git a/arch/sparc/vdso/vma.c b/arch/sparc/vdso/vma.c index f51595f861b8..5eaff3c1aa0c 100644 --- a/arch/sparc/vdso/vma.c +++ b/arch/sparc/vdso/vma.c @@ -262,7 +262,9 @@ static __init int vdso_setup(char *s) unsigned long val; err = kstrtoul(s, 10, &val); + if (err) + return err; vdso_enabled = val; - return err; + return 0; } __setup("vdso=", vdso_setup); diff --git a/arch/um/drivers/ubd_kern.c b/arch/um/drivers/ubd_kern.c index 83c470364dfb..74c002ddc0ce 100644 --- a/arch/um/drivers/ubd_kern.c +++ b/arch/um/drivers/ubd_kern.c @@ -23,6 +23,7 @@ #include <linux/module.h> #include <linux/init.h> #include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/ata.h> #include <linux/hdreg.h> #include <linux/cdrom.h> @@ -142,7 +143,6 @@ struct cow { #define MAX_SG 64 struct ubd { - struct list_head restart; /* name (and fd, below) of the file opened for writing, either the * backing or the cow file. */ char *file; @@ -156,11 +156,8 @@ struct ubd { struct cow cow; struct platform_device pdev; struct request_queue *queue; + struct blk_mq_tag_set tag_set; spinlock_t lock; - struct scatterlist sg[MAX_SG]; - struct request *request; - int start_sg, end_sg; - sector_t rq_pos; }; #define DEFAULT_COW { \ @@ -182,10 +179,6 @@ struct ubd { .shared = 0, \ .cow = DEFAULT_COW, \ .lock = __SPIN_LOCK_UNLOCKED(ubd_devs.lock), \ - .request = NULL, \ - .start_sg = 0, \ - .end_sg = 0, \ - .rq_pos = 0, \ } /* Protected by ubd_lock */ @@ -196,6 +189,9 @@ static int fake_ide = 0; static struct proc_dir_entry *proc_ide_root = NULL; static struct proc_dir_entry *proc_ide = NULL; +static blk_status_t ubd_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd); + static void make_proc_ide(void) { proc_ide_root = proc_mkdir("ide", NULL); @@ -436,11 +432,8 @@ __uml_help(udb_setup, " in the boot output.\n\n" ); -static void do_ubd_request(struct request_queue * q); - /* Only changed by ubd_init, which is an initcall. */ static int thread_fd = -1; -static LIST_HEAD(restart); /* Function to read several request pointers at a time * handling fractional reads if (and as) needed @@ -498,9 +491,6 @@ static int bulk_req_safe_read( /* Called without dev->lock held, and only in interrupt context. */ static void ubd_handler(void) { - struct ubd *ubd; - struct list_head *list, *next_ele; - unsigned long flags; int n; int count; @@ -520,23 +510,17 @@ static void ubd_handler(void) return; } for (count = 0; count < n/sizeof(struct io_thread_req *); count++) { - blk_end_request( - (*irq_req_buffer)[count]->req, - BLK_STS_OK, - (*irq_req_buffer)[count]->length - ); - kfree((*irq_req_buffer)[count]); + struct io_thread_req *io_req = (*irq_req_buffer)[count]; + int err = io_req->error ? BLK_STS_IOERR : BLK_STS_OK; + + if (!blk_update_request(io_req->req, err, io_req->length)) + __blk_mq_end_request(io_req->req, err); + + kfree(io_req); } } - reactivate_fd(thread_fd, UBD_IRQ); - list_for_each_safe(list, next_ele, &restart){ - ubd = container_of(list, struct ubd, restart); - list_del_init(&ubd->restart); - spin_lock_irqsave(&ubd->lock, flags); - do_ubd_request(ubd->queue); - spin_unlock_irqrestore(&ubd->lock, flags); - } + reactivate_fd(thread_fd, UBD_IRQ); } static irqreturn_t ubd_intr(int irq, void *dev) @@ -857,6 +841,7 @@ static void ubd_device_release(struct device *dev) struct ubd *ubd_dev = dev_get_drvdata(dev); blk_cleanup_queue(ubd_dev->queue); + blk_mq_free_tag_set(&ubd_dev->tag_set); *ubd_dev = ((struct ubd) DEFAULT_UBD); } @@ -891,7 +876,7 @@ static int ubd_disk_register(int major, u64 size, int unit, disk->private_data = &ubd_devs[unit]; disk->queue = ubd_devs[unit].queue; - device_add_disk(parent, disk); + device_add_disk(parent, disk, NULL); *disk_out = disk; return 0; @@ -899,6 +884,10 @@ static int ubd_disk_register(int major, u64 size, int unit, #define ROUND_BLOCK(n) ((n + ((1 << 9) - 1)) & (-1 << 9)) +static const struct blk_mq_ops ubd_mq_ops = { + .queue_rq = ubd_queue_rq, +}; + static int ubd_add(int n, char **error_out) { struct ubd *ubd_dev = &ubd_devs[n]; @@ -915,15 +904,23 @@ static int ubd_add(int n, char **error_out) ubd_dev->size = ROUND_BLOCK(ubd_dev->size); - INIT_LIST_HEAD(&ubd_dev->restart); - sg_init_table(ubd_dev->sg, MAX_SG); + ubd_dev->tag_set.ops = &ubd_mq_ops; + ubd_dev->tag_set.queue_depth = 64; + ubd_dev->tag_set.numa_node = NUMA_NO_NODE; + ubd_dev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; + ubd_dev->tag_set.driver_data = ubd_dev; + ubd_dev->tag_set.nr_hw_queues = 1; - err = -ENOMEM; - ubd_dev->queue = blk_init_queue(do_ubd_request, &ubd_dev->lock); - if (ubd_dev->queue == NULL) { - *error_out = "Failed to initialize device queue"; + err = blk_mq_alloc_tag_set(&ubd_dev->tag_set); + if (err) goto out; + + ubd_dev->queue = blk_mq_init_queue(&ubd_dev->tag_set); + if (IS_ERR(ubd_dev->queue)) { + err = PTR_ERR(ubd_dev->queue); + goto out_cleanup; } + ubd_dev->queue->queuedata = ubd_dev; blk_queue_write_cache(ubd_dev->queue, true, false); @@ -931,7 +928,7 @@ static int ubd_add(int n, char **error_out) err = ubd_disk_register(UBD_MAJOR, ubd_dev->size, n, &ubd_gendisk[n]); if(err){ *error_out = "Failed to register device"; - goto out_cleanup; + goto out_cleanup_tags; } if (fake_major != UBD_MAJOR) @@ -949,6 +946,8 @@ static int ubd_add(int n, char **error_out) out: return err; +out_cleanup_tags: + blk_mq_free_tag_set(&ubd_dev->tag_set); out_cleanup: blk_cleanup_queue(ubd_dev->queue); goto out; @@ -1290,123 +1289,82 @@ static void cowify_req(struct io_thread_req *req, unsigned long *bitmap, req->bitmap_words, bitmap_len); } -/* Called with dev->lock held */ -static void prepare_request(struct request *req, struct io_thread_req *io_req, - unsigned long long offset, int page_offset, - int len, struct page *page) +static int ubd_queue_one_vec(struct blk_mq_hw_ctx *hctx, struct request *req, + u64 off, struct bio_vec *bvec) { - struct gendisk *disk = req->rq_disk; - struct ubd *ubd_dev = disk->private_data; - - io_req->req = req; - io_req->fds[0] = (ubd_dev->cow.file != NULL) ? ubd_dev->cow.fd : - ubd_dev->fd; - io_req->fds[1] = ubd_dev->fd; - io_req->cow_offset = -1; - io_req->offset = offset; - io_req->length = len; - io_req->error = 0; - io_req->sector_mask = 0; - - io_req->op = (rq_data_dir(req) == READ) ? UBD_READ : UBD_WRITE; - io_req->offsets[0] = 0; - io_req->offsets[1] = ubd_dev->cow.data_offset; - io_req->buffer = page_address(page) + page_offset; - io_req->sectorsize = 1 << 9; - - if(ubd_dev->cow.file != NULL) - cowify_req(io_req, ubd_dev->cow.bitmap, - ubd_dev->cow.bitmap_offset, ubd_dev->cow.bitmap_len); - -} + struct ubd *dev = hctx->queue->queuedata; + struct io_thread_req *io_req; + int ret; -/* Called with dev->lock held */ -static void prepare_flush_request(struct request *req, - struct io_thread_req *io_req) -{ - struct gendisk *disk = req->rq_disk; - struct ubd *ubd_dev = disk->private_data; + io_req = kmalloc(sizeof(struct io_thread_req), GFP_ATOMIC); + if (!io_req) + return -ENOMEM; io_req->req = req; - io_req->fds[0] = (ubd_dev->cow.file != NULL) ? ubd_dev->cow.fd : - ubd_dev->fd; - io_req->op = UBD_FLUSH; -} + if (dev->cow.file) + io_req->fds[0] = dev->cow.fd; + else + io_req->fds[0] = dev->fd; -static bool submit_request(struct io_thread_req *io_req, struct ubd *dev) -{ - int n = os_write_file(thread_fd, &io_req, - sizeof(io_req)); - if (n != sizeof(io_req)) { - if (n != -EAGAIN) - printk("write to io thread failed, " - "errno = %d\n", -n); - else if (list_empty(&dev->restart)) - list_add(&dev->restart, &restart); + if (req_op(req) == REQ_OP_FLUSH) { + io_req->op = UBD_FLUSH; + } else { + io_req->fds[1] = dev->fd; + io_req->cow_offset = -1; + io_req->offset = off; + io_req->length = bvec->bv_len; + io_req->error = 0; + io_req->sector_mask = 0; + + io_req->op = rq_data_dir(req) == READ ? UBD_READ : UBD_WRITE; + io_req->offsets[0] = 0; + io_req->offsets[1] = dev->cow.data_offset; + io_req->buffer = page_address(bvec->bv_page) + bvec->bv_offset; + io_req->sectorsize = 1 << 9; + + if (dev->cow.file) { + cowify_req(io_req, dev->cow.bitmap, + dev->cow.bitmap_offset, dev->cow.bitmap_len); + } + } + ret = os_write_file(thread_fd, &io_req, sizeof(io_req)); + if (ret != sizeof(io_req)) { + if (ret != -EAGAIN) + pr_err("write to io thread failed: %d\n", -ret); kfree(io_req); - return false; } - return true; + + return ret; } -/* Called with dev->lock held */ -static void do_ubd_request(struct request_queue *q) +static blk_status_t ubd_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { - struct io_thread_req *io_req; - struct request *req; - - while(1){ - struct ubd *dev = q->queuedata; - if(dev->request == NULL){ - struct request *req = blk_fetch_request(q); - if(req == NULL) - return; - - dev->request = req; - dev->rq_pos = blk_rq_pos(req); - dev->start_sg = 0; - dev->end_sg = blk_rq_map_sg(q, req, dev->sg); - } - - req = dev->request; + struct request *req = bd->rq; + int ret = 0; - if (req_op(req) == REQ_OP_FLUSH) { - io_req = kmalloc(sizeof(struct io_thread_req), - GFP_ATOMIC); - if (io_req == NULL) { - if (list_empty(&dev->restart)) - list_add(&dev->restart, &restart); - return; - } - prepare_flush_request(req, io_req); - if (submit_request(io_req, dev) == false) - return; - } + blk_mq_start_request(req); - while(dev->start_sg < dev->end_sg){ - struct scatterlist *sg = &dev->sg[dev->start_sg]; - - io_req = kmalloc(sizeof(struct io_thread_req), - GFP_ATOMIC); - if(io_req == NULL){ - if(list_empty(&dev->restart)) - list_add(&dev->restart, &restart); - return; - } - prepare_request(req, io_req, - (unsigned long long)dev->rq_pos << 9, - sg->offset, sg->length, sg_page(sg)); - - if (submit_request(io_req, dev) == false) - return; - - dev->rq_pos += sg->length >> 9; - dev->start_sg++; + if (req_op(req) == REQ_OP_FLUSH) { + ret = ubd_queue_one_vec(hctx, req, 0, NULL); + } else { + struct req_iterator iter; + struct bio_vec bvec; + u64 off = (u64)blk_rq_pos(req) << 9; + + rq_for_each_segment(bvec, req, iter) { + ret = ubd_queue_one_vec(hctx, req, off, &bvec); + if (ret < 0) + goto out; + off += bvec.bv_len; } - dev->end_sg = 0; - dev->request = NULL; } +out: + if (ret < 0) { + blk_mq_requeue_request(req, true); + } + return BLK_STS_OK; } static int ubd_getgeo(struct block_device *bdev, struct hd_geometry *geo) diff --git a/arch/unicore32/Kconfig b/arch/unicore32/Kconfig index 60eae744d8fd..3a3b40f79558 100644 --- a/arch/unicore32/Kconfig +++ b/arch/unicore32/Kconfig @@ -4,6 +4,7 @@ config UNICORE32 select ARCH_HAS_DEVMEM_IS_ALLOWED select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO + select DMA_DIRECT_OPS select HAVE_MEMBLOCK select HAVE_GENERIC_DMA_COHERENT select HAVE_KERNEL_GZIP @@ -20,7 +21,6 @@ config UNICORE32 select GENERIC_IOMAP select MODULES_USE_ELF_REL select NEED_DMA_MAP_STATE - select SWIOTLB help UniCore-32 is 32-bit Instruction Set Architecture, including a series of low-power-consumption RISC chip diff --git a/arch/unicore32/include/asm/Kbuild b/arch/unicore32/include/asm/Kbuild index bfc7abe77905..1372553dc0a9 100644 --- a/arch/unicore32/include/asm/Kbuild +++ b/arch/unicore32/include/asm/Kbuild @@ -4,6 +4,7 @@ generic-y += compat.h generic-y += current.h generic-y += device.h generic-y += div64.h +generic-y += dma-mapping.h generic-y += emergency-restart.h generic-y += exec.h generic-y += extable.h diff --git a/arch/unicore32/include/asm/dma-mapping.h b/arch/unicore32/include/asm/dma-mapping.h deleted file mode 100644 index 790bc2ef4af2..000000000000 --- a/arch/unicore32/include/asm/dma-mapping.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * linux/arch/unicore32/include/asm/dma-mapping.h - * - * Code specific to PKUnity SoC and UniCore ISA - * - * Copyright (C) 2001-2010 GUAN Xue-tao - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ -#ifndef __UNICORE_DMA_MAPPING_H__ -#define __UNICORE_DMA_MAPPING_H__ - -#include <linux/swiotlb.h> - -static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) -{ - return &swiotlb_dma_ops; -} - -#endif diff --git a/arch/unicore32/mm/init.c b/arch/unicore32/mm/init.c index f4950fbfe574..5f72a8d1d953 100644 --- a/arch/unicore32/mm/init.c +++ b/arch/unicore32/mm/init.c @@ -234,9 +234,6 @@ void __init bootmem_init(void) uc32_bootmem_init(min, max_low); -#ifdef CONFIG_SWIOTLB - swiotlb_init(1); -#endif /* * Sparsemem tries to allocate bootmem in memory_present(), * so must be done after the fixed reservations diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 28764dacf018..466f66c8a7f8 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -37,6 +37,7 @@ KBUILD_CFLAGS += $(call cc-option,-ffreestanding) KBUILD_CFLAGS += $(call cc-option,-fno-stack-protector) KBUILD_CFLAGS += $(call cc-disable-warning, address-of-packed-member) KBUILD_CFLAGS += $(call cc-disable-warning, gnu) +KBUILD_CFLAGS += -Wno-pointer-sign KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ GCOV_PROFILE := n diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index 2767c625a52c..fbbf1ba57ec6 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -389,6 +389,13 @@ * that register for the time this macro runs */ + /* + * The high bits of the CS dword (__csh) are used for + * CS_FROM_ENTRY_STACK and CS_FROM_USER_CR3. Clear them in case + * hardware didn't do this for us. + */ + andl $(0x0000ffff), PT_CS(%esp) + /* Are we on the entry stack? Bail out if not! */ movl PER_CPU_VAR(cpu_entry_area), %ecx addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx @@ -407,12 +414,6 @@ /* Load top of task-stack into %edi */ movl TSS_entry2task_stack(%edi), %edi - /* - * Clear unused upper bits of the dword containing the word-sized CS - * slot in pt_regs in case hardware didn't clear it for us. - */ - andl $(0x0000ffff), PT_CS(%esp) - /* Special case - entry from kernel mode via entry stack */ #ifdef CONFIG_VM86 movl PT_EFLAGS(%esp), %ecx # mix EFLAGS and CS diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 957dfb693ecc..f95dcb209fdf 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -1187,6 +1187,16 @@ ENTRY(paranoid_entry) xorl %ebx, %ebx 1: + /* + * Always stash CR3 in %r14. This value will be restored, + * verbatim, at exit. Needed if paranoid_entry interrupted + * another entry that already switched to the user CR3 value + * but has not yet returned to userspace. + * + * This is also why CS (stashed in the "iret frame" by the + * hardware at entry) can not be used: this may be a return + * to kernel code, but with a user CR3 value. + */ SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14 ret @@ -1211,11 +1221,13 @@ ENTRY(paranoid_exit) testl %ebx, %ebx /* swapgs needed? */ jnz .Lparanoid_exit_no_swapgs TRACE_IRQS_IRETQ + /* Always restore stashed CR3 value (see paranoid_entry) */ RESTORE_CR3 scratch_reg=%rbx save_reg=%r14 SWAPGS_UNSAFE_STACK jmp .Lparanoid_exit_restore .Lparanoid_exit_no_swapgs: TRACE_IRQS_IRETQ_DEBUG + /* Always restore stashed CR3 value (see paranoid_entry) */ RESTORE_CR3 scratch_reg=%rbx save_reg=%r14 .Lparanoid_exit_restore: jmp restore_regs_and_return_to_kernel @@ -1626,6 +1638,7 @@ end_repeat_nmi: movq $-1, %rsi call do_nmi + /* Always restore stashed CR3 value (see paranoid_entry) */ RESTORE_CR3 scratch_reg=%r15 save_reg=%r14 testl %ebx, %ebx /* swapgs needed? */ diff --git a/arch/x86/include/asm/fpu/internal.h b/arch/x86/include/asm/fpu/internal.h index a38bf5a1e37a..69dcdf195b61 100644 --- a/arch/x86/include/asm/fpu/internal.h +++ b/arch/x86/include/asm/fpu/internal.h @@ -528,7 +528,7 @@ static inline void fpregs_activate(struct fpu *fpu) static inline void switch_fpu_prepare(struct fpu *old_fpu, int cpu) { - if (old_fpu->initialized) { + if (static_cpu_has(X86_FEATURE_FPU) && old_fpu->initialized) { if (!copy_fpregs_to_fpstate(old_fpu)) old_fpu->last_cpu = -1; else diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h index 6de64840dd22..9a92a3ac2ac5 100644 --- a/arch/x86/include/asm/io.h +++ b/arch/x86/include/asm/io.h @@ -369,18 +369,6 @@ extern void __iomem *ioremap_wt(resource_size_t offset, unsigned long size); extern bool is_early_ioremap_ptep(pte_t *ptep); -#ifdef CONFIG_XEN -#include <xen/xen.h> -struct bio_vec; - -extern bool xen_biovec_phys_mergeable(const struct bio_vec *vec1, - const struct bio_vec *vec2); - -#define BIOVEC_PHYS_MERGEABLE(vec1, vec2) \ - (__BIOVEC_PHYS_MERGEABLE(vec1, vec2) && \ - (!xen_domain() || xen_biovec_phys_mergeable(vec1, vec2))) -#endif /* CONFIG_XEN */ - #define IO_SPACE_LIMIT 0xffff #include <asm-generic/io.h> diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index e9202a0de8f0..1a19d11cfbbd 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -185,22 +185,22 @@ do { \ typeof(var) pfo_ret__; \ switch (sizeof(var)) { \ case 1: \ - asm(op "b "__percpu_arg(1)",%0" \ + asm volatile(op "b "__percpu_arg(1)",%0"\ : "=q" (pfo_ret__) \ : "m" (var)); \ break; \ case 2: \ - asm(op "w "__percpu_arg(1)",%0" \ + asm volatile(op "w "__percpu_arg(1)",%0"\ : "=r" (pfo_ret__) \ : "m" (var)); \ break; \ case 4: \ - asm(op "l "__percpu_arg(1)",%0" \ + asm volatile(op "l "__percpu_arg(1)",%0"\ : "=r" (pfo_ret__) \ : "m" (var)); \ break; \ case 8: \ - asm(op "q "__percpu_arg(1)",%0" \ + asm volatile(op "q "__percpu_arg(1)",%0"\ : "=r" (pfo_ret__) \ : "m" (var)); \ break; \ diff --git a/arch/x86/include/asm/xen/events.h b/arch/x86/include/asm/xen/events.h index d383140e1dc8..068d9b067c83 100644 --- a/arch/x86/include/asm/xen/events.h +++ b/arch/x86/include/asm/xen/events.h @@ -2,6 +2,8 @@ #ifndef _ASM_X86_XEN_EVENTS_H #define _ASM_X86_XEN_EVENTS_H +#include <xen/xen.h> + enum ipi_vector { XEN_RESCHEDULE_VECTOR, XEN_CALL_FUNCTION_VECTOR, diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c index f299d8a479bb..3f9d1b4019bb 100644 --- a/arch/x86/kernel/amd_gart_64.c +++ b/arch/x86/kernel/amd_gart_64.c @@ -482,7 +482,7 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, { void *vaddr; - vaddr = dma_direct_alloc(dev, size, dma_addr, flag, attrs); + vaddr = dma_direct_alloc_pages(dev, size, dma_addr, flag, attrs); if (!vaddr || !force_iommu || dev->coherent_dma_mask <= DMA_BIT_MASK(24)) return vaddr; @@ -494,7 +494,7 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, goto out_free; return vaddr; out_free: - dma_direct_free(dev, size, vaddr, *dma_addr, attrs); + dma_direct_free_pages(dev, size, vaddr, *dma_addr, attrs); return NULL; } @@ -504,7 +504,7 @@ gart_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, unsigned long attrs) { gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0); - dma_direct_free(dev, size, vaddr, dma_addr, attrs); + dma_direct_free_pages(dev, size, vaddr, dma_addr, attrs); } static int gart_mapping_error(struct device *dev, dma_addr_t dma_addr) diff --git a/arch/x86/kernel/fpu/signal.c b/arch/x86/kernel/fpu/signal.c index 23f1691670b6..61a949d84dfa 100644 --- a/arch/x86/kernel/fpu/signal.c +++ b/arch/x86/kernel/fpu/signal.c @@ -314,7 +314,6 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size) * thread's fpu state, reconstruct fxstate from the fsave * header. Validate and sanitize the copied state. */ - struct fpu *fpu = &tsk->thread.fpu; struct user_i387_ia32_struct env; int err = 0; diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c index 661583662430..71c0b01d93b1 100644 --- a/arch/x86/kernel/pci-swiotlb.c +++ b/arch/x86/kernel/pci-swiotlb.c @@ -42,10 +42,8 @@ IOMMU_INIT_FINISH(pci_swiotlb_detect_override, int __init pci_swiotlb_detect_4gb(void) { /* don't initialize swiotlb if iommu=off (no_iommu=1) */ -#ifdef CONFIG_X86_64 if (!no_iommu && max_possible_pfn > MAX_DMA32_PFN) swiotlb = 1; -#endif /* * If SME is active then swiotlb will be set to 1 so that bounce diff --git a/arch/x86/kernel/time.c b/arch/x86/kernel/time.c index be01328eb755..fddaefc51fb6 100644 --- a/arch/x86/kernel/time.c +++ b/arch/x86/kernel/time.c @@ -25,7 +25,7 @@ #include <asm/time.h> #ifdef CONFIG_X86_64 -__visible volatile unsigned long jiffies __cacheline_aligned = INITIAL_JIFFIES; +__visible volatile unsigned long jiffies __cacheline_aligned_in_smp = INITIAL_JIFFIES; #endif unsigned long profile_pc(struct pt_regs *regs) diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index b52bd2b6cdb4..6d5dc5dabfd7 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -58,7 +58,7 @@ struct cyc2ns { static DEFINE_PER_CPU_ALIGNED(struct cyc2ns, cyc2ns); -void cyc2ns_read_begin(struct cyc2ns_data *data) +void __always_inline cyc2ns_read_begin(struct cyc2ns_data *data) { int seq, idx; @@ -75,7 +75,7 @@ void cyc2ns_read_begin(struct cyc2ns_data *data) } while (unlikely(seq != this_cpu_read(cyc2ns.seq.sequence))); } -void cyc2ns_read_end(void) +void __always_inline cyc2ns_read_end(void) { preempt_enable_notrace(); } @@ -104,7 +104,7 @@ void cyc2ns_read_end(void) * -johnstul@us.ibm.com "math is hard, lets go shopping!" */ -static inline unsigned long long cycles_2_ns(unsigned long long cyc) +static __always_inline unsigned long long cycles_2_ns(unsigned long long cyc) { struct cyc2ns_data data; unsigned long long ns; diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 2eeddd814653..0ca46e03b830 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -5,6 +5,7 @@ #include <linux/kexec.h> #include <linux/slab.h> +#include <xen/xen.h> #include <xen/features.h> #include <xen/page.h> #include <xen/interface/memory.h> diff --git a/arch/x86/xen/enlighten_pvh.c b/arch/x86/xen/enlighten_pvh.c index c85d1a88f476..2a9025343534 100644 --- a/arch/x86/xen/enlighten_pvh.c +++ b/arch/x86/xen/enlighten_pvh.c @@ -11,6 +11,7 @@ #include <asm/xen/interface.h> #include <asm/xen/hypercall.h> +#include <xen/xen.h> #include <xen/interface/memory.h> #include <xen/interface/hvm/start_info.h> diff --git a/arch/x86/xen/platform-pci-unplug.c b/arch/x86/xen/platform-pci-unplug.c index 33a783c77d96..b99585034dd2 100644 --- a/arch/x86/xen/platform-pci-unplug.c +++ b/arch/x86/xen/platform-pci-unplug.c @@ -23,6 +23,7 @@ #include <linux/io.h> #include <linux/export.h> +#include <xen/xen.h> #include <xen/platform_pci.h> #include "xen-ops.h" diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index 95997e6c0696..0972184f3f19 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -3,6 +3,7 @@ #include <linux/interrupt.h> #include <asm/xen/hypercall.h> +#include <xen/xen.h> #include <xen/page.h> #include <xen/interface/xen.h> #include <xen/interface/vcpu.h> diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index b9ad83a0ee5d..ea5d8d03e53b 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -13,7 +13,7 @@ config XTENSA select BUILDTIME_EXTABLE_SORT select CLONE_BACKWARDS select COMMON_CLK - select DMA_NONCOHERENT_OPS + select DMA_DIRECT_OPS select GENERIC_ATOMIC64 select GENERIC_CLOCKEVENTS select GENERIC_IRQ_SHOW diff --git a/block/Kconfig b/block/Kconfig index 1f2469a0123c..f7045aa47edb 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -74,7 +74,6 @@ config BLK_DEV_BSG config BLK_DEV_BSGLIB bool "Block layer SG support v4 helper lib" - default n select BLK_DEV_BSG select BLK_SCSI_REQUEST help @@ -107,7 +106,6 @@ config BLK_DEV_ZONED config BLK_DEV_THROTTLING bool "Block layer bio throttling support" depends on BLK_CGROUP=y - default n ---help--- Block layer bio throttling support. It can be used to limit the IO rate to a device. IO rate policies are per cgroup and @@ -119,7 +117,6 @@ config BLK_DEV_THROTTLING config BLK_DEV_THROTTLING_LOW bool "Block throttling .low limit interface support (EXPERIMENTAL)" depends on BLK_DEV_THROTTLING - default n ---help--- Add .low limit interface for block throttling. The low limit is a best effort limit to prioritize cgroups. Depending on the setting, the limit @@ -130,7 +127,6 @@ config BLK_DEV_THROTTLING_LOW config BLK_CMDLINE_PARSER bool "Block device command line partition parser" - default n ---help--- Enabling this option allows you to specify the partition layout from the kernel boot args. This is typically of use for embedded devices @@ -141,7 +137,6 @@ config BLK_CMDLINE_PARSER config BLK_WBT bool "Enable support for block device writeback throttling" - default n ---help--- Enabling this option enables the block layer to throttle buffered background writeback from the VM, making it more smooth and having @@ -152,7 +147,6 @@ config BLK_WBT config BLK_CGROUP_IOLATENCY bool "Enable support for latency based cgroup IO protection" depends on BLK_CGROUP=y - default n ---help--- Enabling this option enables the .latency interface for IO throttling. The IO controller will attempt to maintain average IO latencies below @@ -163,7 +157,6 @@ config BLK_CGROUP_IOLATENCY config BLK_WBT_SQ bool "Single queue writeback throttling" - default n depends on BLK_WBT ---help--- Enable writeback throttling by default on legacy single queue devices @@ -228,4 +221,7 @@ config BLK_MQ_RDMA depends on BLOCK && INFINIBAND default y +config BLK_PM + def_bool BLOCK && PM + source block/Kconfig.iosched diff --git a/block/Kconfig.iosched b/block/Kconfig.iosched index a4a8914bf7a4..f95a48b0d7b2 100644 --- a/block/Kconfig.iosched +++ b/block/Kconfig.iosched @@ -36,7 +36,6 @@ config IOSCHED_CFQ config CFQ_GROUP_IOSCHED bool "CFQ Group Scheduling support" depends on IOSCHED_CFQ && BLK_CGROUP - default n ---help--- Enable group IO scheduling in CFQ. @@ -82,7 +81,6 @@ config MQ_IOSCHED_KYBER config IOSCHED_BFQ tristate "BFQ I/O scheduler" - default n ---help--- BFQ I/O scheduler for BLK-MQ. BFQ distributes the bandwidth of of the device among all processes according to their weights, @@ -94,7 +92,6 @@ config IOSCHED_BFQ config BFQ_GROUP_IOSCHED bool "BFQ hierarchical scheduling support" depends on IOSCHED_BFQ && BLK_CGROUP - default n ---help--- Enable hierarchical scheduling in BFQ, using the blkio diff --git a/block/Makefile b/block/Makefile index 572b33f32c07..27eac600474f 100644 --- a/block/Makefile +++ b/block/Makefile @@ -37,3 +37,4 @@ obj-$(CONFIG_BLK_WBT) += blk-wbt.o obj-$(CONFIG_BLK_DEBUG_FS) += blk-mq-debugfs.o obj-$(CONFIG_BLK_DEBUG_FS_ZONED)+= blk-mq-debugfs-zoned.o obj-$(CONFIG_BLK_SED_OPAL) += sed-opal.o +obj-$(CONFIG_BLK_PM) += blk-pm.o diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index 9fe5952d117d..d9a7916ff0ab 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -642,7 +642,7 @@ void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio) uint64_t serial_nr; rcu_read_lock(); - serial_nr = bio_blkcg(bio)->css.serial_nr; + serial_nr = __bio_blkcg(bio)->css.serial_nr; /* * Check whether blkcg has changed. The condition may trigger @@ -651,7 +651,7 @@ void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio) if (unlikely(!bfqd) || likely(bic->blkcg_serial_nr == serial_nr)) goto out; - bfqg = __bfq_bic_change_cgroup(bfqd, bic, bio_blkcg(bio)); + bfqg = __bfq_bic_change_cgroup(bfqd, bic, __bio_blkcg(bio)); /* * Update blkg_path for bfq_log_* functions. We cache this * path, and update it here, for the following diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index 653100fb719e..6075100f03a5 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -624,12 +624,13 @@ void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) } /* - * Tell whether there are active queues or groups with differentiated weights. + * Tell whether there are active queues with different weights or + * active groups. */ -static bool bfq_differentiated_weights(struct bfq_data *bfqd) +static bool bfq_varied_queue_weights_or_active_groups(struct bfq_data *bfqd) { /* - * For weights to differ, at least one of the trees must contain + * For queue weights to differ, queue_weights_tree must contain * at least two nodes. */ return (!RB_EMPTY_ROOT(&bfqd->queue_weights_tree) && @@ -637,9 +638,7 @@ static bool bfq_differentiated_weights(struct bfq_data *bfqd) bfqd->queue_weights_tree.rb_node->rb_right) #ifdef CONFIG_BFQ_GROUP_IOSCHED ) || - (!RB_EMPTY_ROOT(&bfqd->group_weights_tree) && - (bfqd->group_weights_tree.rb_node->rb_left || - bfqd->group_weights_tree.rb_node->rb_right) + (bfqd->num_active_groups > 0 #endif ); } @@ -657,26 +656,25 @@ static bool bfq_differentiated_weights(struct bfq_data *bfqd) * 3) all active groups at the same level in the groups tree have the same * number of children. * - * Unfortunately, keeping the necessary state for evaluating exactly the - * above symmetry conditions would be quite complex and time-consuming. - * Therefore this function evaluates, instead, the following stronger - * sub-conditions, for which it is much easier to maintain the needed - * state: + * Unfortunately, keeping the necessary state for evaluating exactly + * the last two symmetry sub-conditions above would be quite complex + * and time consuming. Therefore this function evaluates, instead, + * only the following stronger two sub-conditions, for which it is + * much easier to maintain the needed state: * 1) all active queues have the same weight, - * 2) all active groups have the same weight, - * 3) all active groups have at most one active child each. - * In particular, the last two conditions are always true if hierarchical - * support and the cgroups interface are not enabled, thus no state needs - * to be maintained in this case. + * 2) there are no active groups. + * In particular, the last condition is always true if hierarchical + * support or the cgroups interface are not enabled, thus no state + * needs to be maintained in this case. */ static bool bfq_symmetric_scenario(struct bfq_data *bfqd) { - return !bfq_differentiated_weights(bfqd); + return !bfq_varied_queue_weights_or_active_groups(bfqd); } /* * If the weight-counter tree passed as input contains no counter for - * the weight of the input entity, then add that counter; otherwise just + * the weight of the input queue, then add that counter; otherwise just * increment the existing counter. * * Note that weight-counter trees contain few nodes in mostly symmetric @@ -687,25 +685,25 @@ static bool bfq_symmetric_scenario(struct bfq_data *bfqd) * In most scenarios, the rate at which nodes are created/destroyed * should be low too. */ -void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_entity *entity, +void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq, struct rb_root *root) { + struct bfq_entity *entity = &bfqq->entity; struct rb_node **new = &(root->rb_node), *parent = NULL; /* - * Do not insert if the entity is already associated with a + * Do not insert if the queue is already associated with a * counter, which happens if: - * 1) the entity is associated with a queue, - * 2) a request arrival has caused the queue to become both + * 1) a request arrival has caused the queue to become both * non-weight-raised, and hence change its weight, and * backlogged; in this respect, each of the two events * causes an invocation of this function, - * 3) this is the invocation of this function caused by the + * 2) this is the invocation of this function caused by the * second event. This second invocation is actually useless, * and we handle this fact by exiting immediately. More * efficient or clearer solutions might possibly be adopted. */ - if (entity->weight_counter) + if (bfqq->weight_counter) return; while (*new) { @@ -715,7 +713,7 @@ void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_entity *entity, parent = *new; if (entity->weight == __counter->weight) { - entity->weight_counter = __counter; + bfqq->weight_counter = __counter; goto inc_counter; } if (entity->weight < __counter->weight) @@ -724,66 +722,67 @@ void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_entity *entity, new = &((*new)->rb_right); } - entity->weight_counter = kzalloc(sizeof(struct bfq_weight_counter), - GFP_ATOMIC); + bfqq->weight_counter = kzalloc(sizeof(struct bfq_weight_counter), + GFP_ATOMIC); /* * In the unlucky event of an allocation failure, we just - * exit. This will cause the weight of entity to not be - * considered in bfq_differentiated_weights, which, in its - * turn, causes the scenario to be deemed wrongly symmetric in - * case entity's weight would have been the only weight making - * the scenario asymmetric. On the bright side, no unbalance - * will however occur when entity becomes inactive again (the - * invocation of this function is triggered by an activation - * of entity). In fact, bfq_weights_tree_remove does nothing - * if !entity->weight_counter. + * exit. This will cause the weight of queue to not be + * considered in bfq_varied_queue_weights_or_active_groups, + * which, in its turn, causes the scenario to be deemed + * wrongly symmetric in case bfqq's weight would have been + * the only weight making the scenario asymmetric. On the + * bright side, no unbalance will however occur when bfqq + * becomes inactive again (the invocation of this function + * is triggered by an activation of queue). In fact, + * bfq_weights_tree_remove does nothing if + * !bfqq->weight_counter. */ - if (unlikely(!entity->weight_counter)) + if (unlikely(!bfqq->weight_counter)) return; - entity->weight_counter->weight = entity->weight; - rb_link_node(&entity->weight_counter->weights_node, parent, new); - rb_insert_color(&entity->weight_counter->weights_node, root); + bfqq->weight_counter->weight = entity->weight; + rb_link_node(&bfqq->weight_counter->weights_node, parent, new); + rb_insert_color(&bfqq->weight_counter->weights_node, root); inc_counter: - entity->weight_counter->num_active++; + bfqq->weight_counter->num_active++; } /* - * Decrement the weight counter associated with the entity, and, if the + * Decrement the weight counter associated with the queue, and, if the * counter reaches 0, remove the counter from the tree. * See the comments to the function bfq_weights_tree_add() for considerations * about overhead. */ void __bfq_weights_tree_remove(struct bfq_data *bfqd, - struct bfq_entity *entity, + struct bfq_queue *bfqq, struct rb_root *root) { - if (!entity->weight_counter) + if (!bfqq->weight_counter) return; - entity->weight_counter->num_active--; - if (entity->weight_counter->num_active > 0) + bfqq->weight_counter->num_active--; + if (bfqq->weight_counter->num_active > 0) goto reset_entity_pointer; - rb_erase(&entity->weight_counter->weights_node, root); - kfree(entity->weight_counter); + rb_erase(&bfqq->weight_counter->weights_node, root); + kfree(bfqq->weight_counter); reset_entity_pointer: - entity->weight_counter = NULL; + bfqq->weight_counter = NULL; } /* - * Invoke __bfq_weights_tree_remove on bfqq and all its inactive - * parent entities. + * Invoke __bfq_weights_tree_remove on bfqq and decrement the number + * of active groups for each queue's inactive parent entity. */ void bfq_weights_tree_remove(struct bfq_data *bfqd, struct bfq_queue *bfqq) { struct bfq_entity *entity = bfqq->entity.parent; - __bfq_weights_tree_remove(bfqd, &bfqq->entity, + __bfq_weights_tree_remove(bfqd, bfqq, &bfqd->queue_weights_tree); for_each_entity(entity) { @@ -797,17 +796,13 @@ void bfq_weights_tree_remove(struct bfq_data *bfqd, * next_in_service for details on why * in_service_entity must be checked too). * - * As a consequence, the weight of entity is - * not to be removed. In addition, if entity - * is active, then its parent entities are - * active as well, and thus their weights are - * not to be removed either. In the end, this - * loop must stop here. + * As a consequence, its parent entities are + * active as well, and thus this loop must + * stop here. */ break; } - __bfq_weights_tree_remove(bfqd, entity, - &bfqd->group_weights_tree); + bfqd->num_active_groups--; } } @@ -3182,6 +3177,13 @@ static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd, jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4); } +static bool bfq_bfqq_injectable(struct bfq_queue *bfqq) +{ + return BFQQ_SEEKY(bfqq) && bfqq->wr_coeff == 1 && + blk_queue_nonrot(bfqq->bfqd->queue) && + bfqq->bfqd->hw_tag; +} + /** * bfq_bfqq_expire - expire a queue. * @bfqd: device owning the queue. @@ -3291,6 +3293,8 @@ void bfq_bfqq_expire(struct bfq_data *bfqd, if (ref == 1) /* bfqq is gone, no more actions on it */ return; + bfqq->injected_service = 0; + /* mark bfqq as waiting a request only if a bic still points to it */ if (!bfq_bfqq_busy(bfqq) && reason != BFQQE_BUDGET_TIMEOUT && @@ -3497,9 +3501,11 @@ static bool bfq_better_to_idle(struct bfq_queue *bfqq) * symmetric scenario where: * (i) each of these processes must get the same throughput as * the others; - * (ii) all these processes have the same I/O pattern - (either sequential or random). - * In fact, in such a scenario, the drive will tend to treat + * (ii) the I/O of each process has the same properties, in + * terms of locality (sequential or random), direction + * (reads or writes), request sizes, greediness + * (from I/O-bound to sporadic), and so on. + * In fact, in such a scenario, the drive tends to treat * the requests of each of these processes in about the same * way as the requests of the others, and thus to provide * each of these processes with about the same throughput @@ -3508,18 +3514,50 @@ static bool bfq_better_to_idle(struct bfq_queue *bfqq) * certainly needed to guarantee that bfqq receives its * assigned fraction of the device throughput (see [1] for * details). + * The problem is that idling may significantly reduce + * throughput with certain combinations of types of I/O and + * devices. An important example is sync random I/O, on flash + * storage with command queueing. So, unless bfqq falls in the + * above cases where idling also boosts throughput, it would + * be important to check conditions (i) and (ii) accurately, + * so as to avoid idling when not strictly needed for service + * guarantees. + * + * Unfortunately, it is extremely difficult to thoroughly + * check condition (ii). And, in case there are active groups, + * it becomes very difficult to check condition (i) too. In + * fact, if there are active groups, then, for condition (i) + * to become false, it is enough that an active group contains + * more active processes or sub-groups than some other active + * group. We address this issue with the following bi-modal + * behavior, implemented in the function + * bfq_symmetric_scenario(). * - * We address this issue by controlling, actually, only the - * symmetry sub-condition (i), i.e., provided that - * sub-condition (i) holds, idling is not performed, - * regardless of whether sub-condition (ii) holds. In other - * words, only if sub-condition (i) holds, then idling is + * If there are active groups, then the scenario is tagged as + * asymmetric, conservatively, without checking any of the + * conditions (i) and (ii). So the device is idled for bfqq. + * This behavior matches also the fact that groups are created + * exactly if controlling I/O (to preserve bandwidth and + * latency guarantees) is a primary concern. + * + * On the opposite end, if there are no active groups, then + * only condition (i) is actually controlled, i.e., provided + * that condition (i) holds, idling is not performed, + * regardless of whether condition (ii) holds. In other words, + * only if condition (i) does not hold, then idling is * allowed, and the device tends to be prevented from queueing - * many requests, possibly of several processes. The reason - * for not controlling also sub-condition (ii) is that we - * exploit preemption to preserve guarantees in case of - * symmetric scenarios, even if (ii) does not hold, as - * explained in the next two paragraphs. + * many requests, possibly of several processes. Since there + * are no active groups, then, to control condition (i) it is + * enough to check whether all active queues have the same + * weight. + * + * Not checking condition (ii) evidently exposes bfqq to the + * risk of getting less throughput than its fair share. + * However, for queues with the same weight, a further + * mechanism, preemption, mitigates or even eliminates this + * problem. And it does so without consequences on overall + * throughput. This mechanism and its benefits are explained + * in the next three paragraphs. * * Even if a queue, say Q, is expired when it remains idle, Q * can still preempt the new in-service queue if the next @@ -3533,11 +3571,7 @@ static bool bfq_better_to_idle(struct bfq_queue *bfqq) * idling allows the internal queues of the device to contain * many requests, and thus to reorder requests, we can rather * safely assume that the internal scheduler still preserves a - * minimum of mid-term fairness. The motivation for using - * preemption instead of idling is that, by not idling, - * service guarantees are preserved without minimally - * sacrificing throughput. In other words, both a high - * throughput and its desired distribution are obtained. + * minimum of mid-term fairness. * * More precisely, this preemption-based, idleless approach * provides fairness in terms of IOPS, and not sectors per @@ -3556,22 +3590,27 @@ static bool bfq_better_to_idle(struct bfq_queue *bfqq) * 1024/8 times as high as the service received by the other * queue. * - * On the other hand, device idling is performed, and thus - * pure sector-domain guarantees are provided, for the - * following queues, which are likely to need stronger - * throughput guarantees: weight-raised queues, and queues - * with a higher weight than other queues. When such queues - * are active, sub-condition (i) is false, which triggers - * device idling. + * The motivation for using preemption instead of idling (for + * queues with the same weight) is that, by not idling, + * service guarantees are preserved (completely or at least in + * part) without minimally sacrificing throughput. And, if + * there is no active group, then the primary expectation for + * this device is probably a high throughput. * - * According to the above considerations, the next variable is - * true (only) if sub-condition (i) holds. To compute the - * value of this variable, we not only use the return value of - * the function bfq_symmetric_scenario(), but also check - * whether bfqq is being weight-raised, because - * bfq_symmetric_scenario() does not take into account also - * weight-raised queues (see comments on - * bfq_weights_tree_add()). + * We are now left only with explaining the additional + * compound condition that is checked below for deciding + * whether the scenario is asymmetric. To explain this + * compound condition, we need to add that the function + * bfq_symmetric_scenario checks the weights of only + * non-weight-raised queues, for efficiency reasons (see + * comments on bfq_weights_tree_add()). Then the fact that + * bfqq is weight-raised is checked explicitly here. More + * precisely, the compound condition below takes into account + * also the fact that, even if bfqq is being weight-raised, + * the scenario is still symmetric if all active queues happen + * to be weight-raised. Actually, we should be even more + * precise here, and differentiate between interactive weight + * raising and soft real-time weight raising. * * As a side note, it is worth considering that the above * device-idling countermeasures may however fail in the @@ -3583,7 +3622,8 @@ static bool bfq_better_to_idle(struct bfq_queue *bfqq) * to let requests be served in the desired order until all * the requests already queued in the device have been served. */ - asymmetric_scenario = bfqq->wr_coeff > 1 || + asymmetric_scenario = (bfqq->wr_coeff > 1 && + bfqd->wr_busy_queues < bfqd->busy_queues) || !bfq_symmetric_scenario(bfqd); /* @@ -3629,6 +3669,30 @@ static bool bfq_bfqq_must_idle(struct bfq_queue *bfqq) return RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_better_to_idle(bfqq); } +static struct bfq_queue *bfq_choose_bfqq_for_injection(struct bfq_data *bfqd) +{ + struct bfq_queue *bfqq; + + /* + * A linear search; but, with a high probability, very few + * steps are needed to find a candidate queue, i.e., a queue + * with enough budget left for its next request. In fact: + * - BFQ dynamically updates the budget of every queue so as + * to accommodate the expected backlog of the queue; + * - if a queue gets all its requests dispatched as injected + * service, then the queue is removed from the active list + * (and re-added only if it gets new requests, but with + * enough budget for its new backlog). + */ + list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) + if (!RB_EMPTY_ROOT(&bfqq->sort_list) && + bfq_serv_to_charge(bfqq->next_rq, bfqq) <= + bfq_bfqq_budget_left(bfqq)) + return bfqq; + + return NULL; +} + /* * Select a queue for service. If we have a current queue in service, * check whether to continue servicing it, or retrieve and set a new one. @@ -3710,10 +3774,19 @@ check_queue: * No requests pending. However, if the in-service queue is idling * for a new request, or has requests waiting for a completion and * may idle after their completion, then keep it anyway. + * + * Yet, to boost throughput, inject service from other queues if + * possible. */ if (bfq_bfqq_wait_request(bfqq) || (bfqq->dispatched != 0 && bfq_better_to_idle(bfqq))) { - bfqq = NULL; + if (bfq_bfqq_injectable(bfqq) && + bfqq->injected_service * bfqq->inject_coeff < + bfqq->entity.service * 10) + bfqq = bfq_choose_bfqq_for_injection(bfqd); + else + bfqq = NULL; + goto keep_queue; } @@ -3803,6 +3876,14 @@ static struct request *bfq_dispatch_rq_from_bfqq(struct bfq_data *bfqd, bfq_dispatch_remove(bfqd->queue, rq); + if (bfqq != bfqd->in_service_queue) { + if (likely(bfqd->in_service_queue)) + bfqd->in_service_queue->injected_service += + bfq_serv_to_charge(rq, bfqq); + + goto return_rq; + } + /* * If weight raising has to terminate for bfqq, then next * function causes an immediate update of bfqq's weight, @@ -3821,13 +3902,12 @@ static struct request *bfq_dispatch_rq_from_bfqq(struct bfq_data *bfqd, * belongs to CLASS_IDLE and other queues are waiting for * service. */ - if (bfqd->busy_queues > 1 && bfq_class_idle(bfqq)) - goto expire; - - return rq; + if (!(bfqd->busy_queues > 1 && bfq_class_idle(bfqq))) + goto return_rq; -expire: bfq_bfqq_expire(bfqd, bfqq, false, BFQQE_BUDGET_EXHAUSTED); + +return_rq: return rq; } @@ -4232,6 +4312,13 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfq_mark_bfqq_has_short_ttime(bfqq); bfq_mark_bfqq_sync(bfqq); bfq_mark_bfqq_just_created(bfqq); + /* + * Aggressively inject a lot of service: up to 90%. + * This coefficient remains constant during bfqq life, + * but this behavior might be changed, after enough + * testing and tuning. + */ + bfqq->inject_coeff = 1; } else bfq_clear_bfqq_sync(bfqq); @@ -4297,7 +4384,7 @@ static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, rcu_read_lock(); - bfqg = bfq_find_set_group(bfqd, bio_blkcg(bio)); + bfqg = bfq_find_set_group(bfqd, __bio_blkcg(bio)); if (!bfqg) { bfqq = &bfqd->oom_bfqq; goto out; @@ -5330,7 +5417,7 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) bfqd->idle_slice_timer.function = bfq_idle_slice_timer; bfqd->queue_weights_tree = RB_ROOT; - bfqd->group_weights_tree = RB_ROOT; + bfqd->num_active_groups = 0; INIT_LIST_HEAD(&bfqd->active_list); INIT_LIST_HEAD(&bfqd->idle_list); diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h index a8a2e5aca4d4..77651d817ecd 100644 --- a/block/bfq-iosched.h +++ b/block/bfq-iosched.h @@ -108,15 +108,14 @@ struct bfq_sched_data { }; /** - * struct bfq_weight_counter - counter of the number of all active entities + * struct bfq_weight_counter - counter of the number of all active queues * with a given weight. */ struct bfq_weight_counter { - unsigned int weight; /* weight of the entities this counter refers to */ - unsigned int num_active; /* nr of active entities with this weight */ + unsigned int weight; /* weight of the queues this counter refers to */ + unsigned int num_active; /* nr of active queues with this weight */ /* - * Weights tree member (see bfq_data's @queue_weights_tree and - * @group_weights_tree) + * Weights tree member (see bfq_data's @queue_weights_tree) */ struct rb_node weights_node; }; @@ -151,8 +150,6 @@ struct bfq_weight_counter { struct bfq_entity { /* service_tree member */ struct rb_node rb_node; - /* pointer to the weight counter associated with this entity */ - struct bfq_weight_counter *weight_counter; /* * Flag, true if the entity is on a tree (either the active or @@ -266,6 +263,9 @@ struct bfq_queue { /* entity representing this queue in the scheduler */ struct bfq_entity entity; + /* pointer to the weight counter associated with this entity */ + struct bfq_weight_counter *weight_counter; + /* maximum budget allowed from the feedback mechanism */ int max_budget; /* budget expiration (in jiffies) */ @@ -351,6 +351,32 @@ struct bfq_queue { unsigned long split_time; /* time of last split */ unsigned long first_IO_time; /* time of first I/O for this queue */ + + /* max service rate measured so far */ + u32 max_service_rate; + /* + * Ratio between the service received by bfqq while it is in + * service, and the cumulative service (of requests of other + * queues) that may be injected while bfqq is empty but still + * in service. To increase precision, the coefficient is + * measured in tenths of unit. Here are some example of (1) + * ratios, (2) resulting percentages of service injected + * w.r.t. to the total service dispatched while bfqq is in + * service, and (3) corresponding values of the coefficient: + * 1 (50%) -> 10 + * 2 (33%) -> 20 + * 10 (9%) -> 100 + * 9.9 (9%) -> 99 + * 1.5 (40%) -> 15 + * 0.5 (66%) -> 5 + * 0.1 (90%) -> 1 + * + * So, if the coefficient is lower than 10, then + * injected service is more than bfqq service. + */ + unsigned int inject_coeff; + /* amount of service injected in current service slot */ + unsigned int injected_service; }; /** @@ -423,14 +449,9 @@ struct bfq_data { */ struct rb_root queue_weights_tree; /* - * rbtree of non-queue @bfq_entity weight counters, sorted by - * weight. Used to keep track of whether all @bfq_groups have - * the same weight. The tree contains one counter for each - * distinct weight associated to some active @bfq_group (see - * the comments to the functions bfq_weights_tree_[add|remove] - * for further details). + * number of groups with requests still waiting for completion */ - struct rb_root group_weights_tree; + unsigned int num_active_groups; /* * Number of bfq_queues containing requests (including the @@ -825,10 +846,10 @@ struct bfq_queue *bic_to_bfqq(struct bfq_io_cq *bic, bool is_sync); void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, bool is_sync); struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic); void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq); -void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_entity *entity, +void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq, struct rb_root *root); void __bfq_weights_tree_remove(struct bfq_data *bfqd, - struct bfq_entity *entity, + struct bfq_queue *bfqq, struct rb_root *root); void bfq_weights_tree_remove(struct bfq_data *bfqd, struct bfq_queue *bfqq); diff --git a/block/bfq-wf2q.c b/block/bfq-wf2q.c index ae52bff43ce4..476b5a90a5a4 100644 --- a/block/bfq-wf2q.c +++ b/block/bfq-wf2q.c @@ -788,25 +788,29 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st, new_weight = entity->orig_weight * (bfqq ? bfqq->wr_coeff : 1); /* - * If the weight of the entity changes, remove the entity - * from its old weight counter (if there is a counter - * associated with the entity), and add it to the counter - * associated with its new weight. + * If the weight of the entity changes, and the entity is a + * queue, remove the entity from its old weight counter (if + * there is a counter associated with the entity). */ if (prev_weight != new_weight) { - root = bfqq ? &bfqd->queue_weights_tree : - &bfqd->group_weights_tree; - __bfq_weights_tree_remove(bfqd, entity, root); + if (bfqq) { + root = &bfqd->queue_weights_tree; + __bfq_weights_tree_remove(bfqd, bfqq, root); + } else + bfqd->num_active_groups--; } entity->weight = new_weight; /* - * Add the entity to its weights tree only if it is - * not associated with a weight-raised queue. + * Add the entity, if it is not a weight-raised queue, + * to the counter associated with its new weight. */ - if (prev_weight != new_weight && - (bfqq ? bfqq->wr_coeff == 1 : 1)) - /* If we get here, root has been initialized. */ - bfq_weights_tree_add(bfqd, entity, root); + if (prev_weight != new_weight) { + if (bfqq && bfqq->wr_coeff == 1) { + /* If we get here, root has been initialized. */ + bfq_weights_tree_add(bfqd, bfqq, root); + } else + bfqd->num_active_groups++; + } new_st->wsum += entity->weight; @@ -1012,9 +1016,9 @@ static void __bfq_activate_entity(struct bfq_entity *entity, if (!bfq_entity_to_bfqq(entity)) { /* bfq_group */ struct bfq_group *bfqg = container_of(entity, struct bfq_group, entity); + struct bfq_data *bfqd = bfqg->bfqd; - bfq_weights_tree_add(bfqg->bfqd, entity, - &bfqd->group_weights_tree); + bfqd->num_active_groups++; } #endif @@ -1181,10 +1185,17 @@ bool __bfq_deactivate_entity(struct bfq_entity *entity, bool ins_into_idle_tree) st = bfq_entity_service_tree(entity); is_in_service = entity == sd->in_service_entity; - if (is_in_service) { - bfq_calc_finish(entity, entity->service); + bfq_calc_finish(entity, entity->service); + + if (is_in_service) sd->in_service_entity = NULL; - } + else + /* + * Non in-service entity: nobody will take care of + * resetting its service counter on expiration. Do it + * now. + */ + entity->service = 0; if (entity->tree == &st->active) bfq_active_extract(st, entity); @@ -1685,7 +1696,7 @@ void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq) if (!bfqq->dispatched) if (bfqq->wr_coeff == 1) - bfq_weights_tree_add(bfqd, &bfqq->entity, + bfq_weights_tree_add(bfqd, bfqq, &bfqd->queue_weights_tree); if (bfqq->wr_coeff > 1) diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 67b5fb861a51..290af497997b 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -306,6 +306,8 @@ bool bio_integrity_prep(struct bio *bio) if (bio_data_dir(bio) == WRITE) { bio_integrity_process(bio, &bio->bi_iter, bi->profile->generate_fn); + } else { + bip->bio_iter = bio->bi_iter; } return true; @@ -331,20 +333,14 @@ static void bio_integrity_verify_fn(struct work_struct *work) container_of(work, struct bio_integrity_payload, bip_work); struct bio *bio = bip->bip_bio; struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); - struct bvec_iter iter = bio->bi_iter; /* * At the moment verify is called bio's iterator was advanced * during split and completion, we need to rewind iterator to * it's original position. */ - if (bio_rewind_iter(bio, &iter, iter.bi_done)) { - bio->bi_status = bio_integrity_process(bio, &iter, - bi->profile->verify_fn); - } else { - bio->bi_status = BLK_STS_IOERR; - } - + bio->bi_status = bio_integrity_process(bio, &bip->bio_iter, + bi->profile->verify_fn); bio_integrity_free(bio); bio_endio(bio); } diff --git a/block/bio.c b/block/bio.c index 0093bed81c0e..bbfeb4ee2892 100644 --- a/block/bio.c +++ b/block/bio.c @@ -609,7 +609,9 @@ void __bio_clone_fast(struct bio *bio, struct bio *bio_src) bio->bi_iter = bio_src->bi_iter; bio->bi_io_vec = bio_src->bi_io_vec; - bio_clone_blkcg_association(bio, bio_src); + bio_clone_blkg_association(bio, bio_src); + + blkcg_bio_issue_init(bio); } EXPORT_SYMBOL(__bio_clone_fast); @@ -729,7 +731,7 @@ int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page } /* If we may be able to merge these biovecs, force a recount */ - if (bio->bi_vcnt > 1 && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec))) + if (bio->bi_vcnt > 1 && biovec_phys_mergeable(q, bvec - 1, bvec)) bio_clear_flag(bio, BIO_SEG_VALID); done: @@ -827,6 +829,8 @@ int bio_add_page(struct bio *bio, struct page *page, } EXPORT_SYMBOL(bio_add_page); +#define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *)) + /** * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio * @bio: bio to add pages to @@ -839,38 +843,35 @@ EXPORT_SYMBOL(bio_add_page); */ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) { - unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt, idx; + unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; + unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt; struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; struct page **pages = (struct page **)bv; + ssize_t size, left; + unsigned len, i; size_t offset; - ssize_t size; + + /* + * Move page array up in the allocated memory for the bio vecs as far as + * possible so that we can start filling biovecs from the beginning + * without overwriting the temporary page array. + */ + BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2); + pages += entries_left * (PAGE_PTRS_PER_BVEC - 1); size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset); if (unlikely(size <= 0)) return size ? size : -EFAULT; - idx = nr_pages = (size + offset + PAGE_SIZE - 1) / PAGE_SIZE; - /* - * Deep magic below: We need to walk the pinned pages backwards - * because we are abusing the space allocated for the bio_vecs - * for the page array. Because the bio_vecs are larger than the - * page pointers by definition this will always work. But it also - * means we can't use bio_add_page, so any changes to it's semantics - * need to be reflected here as well. - */ - bio->bi_iter.bi_size += size; - bio->bi_vcnt += nr_pages; + for (left = size, i = 0; left > 0; left -= len, i++) { + struct page *page = pages[i]; - while (idx--) { - bv[idx].bv_page = pages[idx]; - bv[idx].bv_len = PAGE_SIZE; - bv[idx].bv_offset = 0; + len = min_t(size_t, PAGE_SIZE - offset, left); + if (WARN_ON_ONCE(bio_add_page(bio, page, len, offset) != len)) + return -EINVAL; + offset = 0; } - bv[0].bv_offset += offset; - bv[0].bv_len -= offset; - bv[nr_pages - 1].bv_len -= nr_pages * PAGE_SIZE - offset - size; - iov_iter_advance(iter, size); return 0; } @@ -1807,7 +1808,6 @@ struct bio *bio_split(struct bio *bio, int sectors, bio_integrity_trim(split); bio_advance(bio, split->bi_iter.bi_size); - bio->bi_iter.bi_done = 0; if (bio_flagged(bio, BIO_TRACE_COMPLETION)) bio_set_flag(split, BIO_TRACE_COMPLETION); @@ -1956,69 +1956,151 @@ EXPORT_SYMBOL(bioset_init_from_src); #ifdef CONFIG_BLK_CGROUP +/** + * bio_associate_blkg - associate a bio with the a blkg + * @bio: target bio + * @blkg: the blkg to associate + * + * This tries to associate @bio with the specified blkg. Association failure + * is handled by walking up the blkg tree. Therefore, the blkg associated can + * be anything between @blkg and the root_blkg. This situation only happens + * when a cgroup is dying and then the remaining bios will spill to the closest + * alive blkg. + * + * A reference will be taken on the @blkg and will be released when @bio is + * freed. + */ +int bio_associate_blkg(struct bio *bio, struct blkcg_gq *blkg) +{ + if (unlikely(bio->bi_blkg)) + return -EBUSY; + bio->bi_blkg = blkg_tryget_closest(blkg); + return 0; +} + +/** + * __bio_associate_blkg_from_css - internal blkg association function + * + * This in the core association function that all association paths rely on. + * A blkg reference is taken which is released upon freeing of the bio. + */ +static int __bio_associate_blkg_from_css(struct bio *bio, + struct cgroup_subsys_state *css) +{ + struct request_queue *q = bio->bi_disk->queue; + struct blkcg_gq *blkg; + int ret; + + rcu_read_lock(); + + if (!css || !css->parent) + blkg = q->root_blkg; + else + blkg = blkg_lookup_create(css_to_blkcg(css), q); + + ret = bio_associate_blkg(bio, blkg); + + rcu_read_unlock(); + return ret; +} + +/** + * bio_associate_blkg_from_css - associate a bio with a specified css + * @bio: target bio + * @css: target css + * + * Associate @bio with the blkg found by combining the css's blkg and the + * request_queue of the @bio. This falls back to the queue's root_blkg if + * the association fails with the css. + */ +int bio_associate_blkg_from_css(struct bio *bio, + struct cgroup_subsys_state *css) +{ + if (unlikely(bio->bi_blkg)) + return -EBUSY; + return __bio_associate_blkg_from_css(bio, css); +} +EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css); + #ifdef CONFIG_MEMCG /** - * bio_associate_blkcg_from_page - associate a bio with the page's blkcg + * bio_associate_blkg_from_page - associate a bio with the page's blkg * @bio: target bio * @page: the page to lookup the blkcg from * - * Associate @bio with the blkcg from @page's owning memcg. This works like - * every other associate function wrt references. + * Associate @bio with the blkg from @page's owning memcg and the respective + * request_queue. If cgroup_e_css returns NULL, fall back to the queue's + * root_blkg. + * + * Note: this must be called after bio has an associated device. */ -int bio_associate_blkcg_from_page(struct bio *bio, struct page *page) +int bio_associate_blkg_from_page(struct bio *bio, struct page *page) { - struct cgroup_subsys_state *blkcg_css; + struct cgroup_subsys_state *css; + int ret; - if (unlikely(bio->bi_css)) + if (unlikely(bio->bi_blkg)) return -EBUSY; if (!page->mem_cgroup) return 0; - blkcg_css = cgroup_get_e_css(page->mem_cgroup->css.cgroup, - &io_cgrp_subsys); - bio->bi_css = blkcg_css; - return 0; + + rcu_read_lock(); + + css = cgroup_e_css(page->mem_cgroup->css.cgroup, &io_cgrp_subsys); + + ret = __bio_associate_blkg_from_css(bio, css); + + rcu_read_unlock(); + return ret; } #endif /* CONFIG_MEMCG */ /** - * bio_associate_blkcg - associate a bio with the specified blkcg + * bio_associate_create_blkg - associate a bio with a blkg from q + * @q: request_queue where bio is going * @bio: target bio - * @blkcg_css: css of the blkcg to associate - * - * Associate @bio with the blkcg specified by @blkcg_css. Block layer will - * treat @bio as if it were issued by a task which belongs to the blkcg. * - * This function takes an extra reference of @blkcg_css which will be put - * when @bio is released. The caller must own @bio and is responsible for - * synchronizing calls to this function. + * Associate @bio with the blkg found from the bio's css and the request_queue. + * If one is not found, bio_lookup_blkg creates the blkg. This falls back to + * the queue's root_blkg if association fails. */ -int bio_associate_blkcg(struct bio *bio, struct cgroup_subsys_state *blkcg_css) +int bio_associate_create_blkg(struct request_queue *q, struct bio *bio) { - if (unlikely(bio->bi_css)) - return -EBUSY; - css_get(blkcg_css); - bio->bi_css = blkcg_css; - return 0; + struct cgroup_subsys_state *css; + int ret = 0; + + /* someone has already associated this bio with a blkg */ + if (bio->bi_blkg) + return ret; + + rcu_read_lock(); + + css = blkcg_css(); + + ret = __bio_associate_blkg_from_css(bio, css); + + rcu_read_unlock(); + return ret; } -EXPORT_SYMBOL_GPL(bio_associate_blkcg); /** - * bio_associate_blkg - associate a bio with the specified blkg + * bio_reassociate_blkg - reassociate a bio with a blkg from q + * @q: request_queue where bio is going * @bio: target bio - * @blkg: the blkg to associate * - * Associate @bio with the blkg specified by @blkg. This is the queue specific - * blkcg information associated with the @bio, a reference will be taken on the - * @blkg and will be freed when the bio is freed. + * When submitting a bio, multiple recursive calls to make_request() may occur. + * This causes the initial associate done in blkcg_bio_issue_check() to be + * incorrect and reference the prior request_queue. This performs reassociation + * when this situation happens. */ -int bio_associate_blkg(struct bio *bio, struct blkcg_gq *blkg) +int bio_reassociate_blkg(struct request_queue *q, struct bio *bio) { - if (unlikely(bio->bi_blkg)) - return -EBUSY; - if (!blkg_try_get(blkg)) - return -ENODEV; - bio->bi_blkg = blkg; - return 0; + if (bio->bi_blkg) { + blkg_put(bio->bi_blkg); + bio->bi_blkg = NULL; + } + + return bio_associate_create_blkg(q, bio); } /** @@ -2031,10 +2113,6 @@ void bio_disassociate_task(struct bio *bio) put_io_context(bio->bi_ioc); bio->bi_ioc = NULL; } - if (bio->bi_css) { - css_put(bio->bi_css); - bio->bi_css = NULL; - } if (bio->bi_blkg) { blkg_put(bio->bi_blkg); bio->bi_blkg = NULL; @@ -2042,16 +2120,16 @@ void bio_disassociate_task(struct bio *bio) } /** - * bio_clone_blkcg_association - clone blkcg association from src to dst bio + * bio_clone_blkg_association - clone blkg association from src to dst bio * @dst: destination bio * @src: source bio */ -void bio_clone_blkcg_association(struct bio *dst, struct bio *src) +void bio_clone_blkg_association(struct bio *dst, struct bio *src) { - if (src->bi_css) - WARN_ON(bio_associate_blkcg(dst, src->bi_css)); + if (src->bi_blkg) + bio_associate_blkg(dst, src->bi_blkg); } -EXPORT_SYMBOL_GPL(bio_clone_blkcg_association); +EXPORT_SYMBOL_GPL(bio_clone_blkg_association); #endif /* CONFIG_BLK_CGROUP */ static void __init biovec_init_slabs(void) diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index c630e02836a8..992da5592c6e 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -84,6 +84,37 @@ static void blkg_free(struct blkcg_gq *blkg) kfree(blkg); } +static void __blkg_release(struct rcu_head *rcu) +{ + struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head); + + percpu_ref_exit(&blkg->refcnt); + + /* release the blkcg and parent blkg refs this blkg has been holding */ + css_put(&blkg->blkcg->css); + if (blkg->parent) + blkg_put(blkg->parent); + + wb_congested_put(blkg->wb_congested); + + blkg_free(blkg); +} + +/* + * A group is RCU protected, but having an rcu lock does not mean that one + * can access all the fields of blkg and assume these are valid. For + * example, don't try to follow throtl_data and request queue links. + * + * Having a reference to blkg under an rcu allows accesses to only values + * local to groups like group stats and group rate limits. + */ +static void blkg_release(struct percpu_ref *ref) +{ + struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt); + + call_rcu(&blkg->rcu_head, __blkg_release); +} + /** * blkg_alloc - allocate a blkg * @blkcg: block cgroup the new blkg is associated with @@ -110,7 +141,6 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q, blkg->q = q; INIT_LIST_HEAD(&blkg->q_node); blkg->blkcg = blkcg; - atomic_set(&blkg->refcnt, 1); /* root blkg uses @q->root_rl, init rl only for !root blkgs */ if (blkcg != &blkcg_root) { @@ -217,6 +247,11 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, blkg_get(blkg->parent); } + ret = percpu_ref_init(&blkg->refcnt, blkg_release, 0, + GFP_NOWAIT | __GFP_NOWARN); + if (ret) + goto err_cancel_ref; + /* invoke per-policy init */ for (i = 0; i < BLKCG_MAX_POLS; i++) { struct blkcg_policy *pol = blkcg_policy[i]; @@ -249,6 +284,8 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, blkg_put(blkg); return ERR_PTR(ret); +err_cancel_ref: + percpu_ref_exit(&blkg->refcnt); err_put_congested: wb_congested_put(wb_congested); err_put_css: @@ -259,7 +296,7 @@ err_free_blkg: } /** - * blkg_lookup_create - lookup blkg, try to create one if not there + * __blkg_lookup_create - lookup blkg, try to create one if not there * @blkcg: blkcg of interest * @q: request_queue of interest * @@ -268,12 +305,11 @@ err_free_blkg: * that all non-root blkg's have access to the parent blkg. This function * should be called under RCU read lock and @q->queue_lock. * - * Returns pointer to the looked up or created blkg on success, ERR_PTR() - * value on error. If @q is dead, returns ERR_PTR(-EINVAL). If @q is not - * dead and bypassing, returns ERR_PTR(-EBUSY). + * Returns the blkg or the closest blkg if blkg_create fails as it walks + * down from root. */ -struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, - struct request_queue *q) +struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg, + struct request_queue *q) { struct blkcg_gq *blkg; @@ -285,7 +321,7 @@ struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, * we shouldn't allow anything to go through for a bypassing queue. */ if (unlikely(blk_queue_bypass(q))) - return ERR_PTR(blk_queue_dying(q) ? -ENODEV : -EBUSY); + return q->root_blkg; blkg = __blkg_lookup(blkcg, q, true); if (blkg) @@ -293,23 +329,58 @@ struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, /* * Create blkgs walking down from blkcg_root to @blkcg, so that all - * non-root blkgs have access to their parents. + * non-root blkgs have access to their parents. Returns the closest + * blkg to the intended blkg should blkg_create() fail. */ while (true) { struct blkcg *pos = blkcg; struct blkcg *parent = blkcg_parent(blkcg); - - while (parent && !__blkg_lookup(parent, q, false)) { + struct blkcg_gq *ret_blkg = q->root_blkg; + + while (parent) { + blkg = __blkg_lookup(parent, q, false); + if (blkg) { + /* remember closest blkg */ + ret_blkg = blkg; + break; + } pos = parent; parent = blkcg_parent(parent); } blkg = blkg_create(pos, q, NULL); - if (pos == blkcg || IS_ERR(blkg)) + if (IS_ERR(blkg)) + return ret_blkg; + if (pos == blkcg) return blkg; } } +/** + * blkg_lookup_create - find or create a blkg + * @blkcg: target block cgroup + * @q: target request_queue + * + * This looks up or creates the blkg representing the unique pair + * of the blkcg and the request_queue. + */ +struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, + struct request_queue *q) +{ + struct blkcg_gq *blkg = blkg_lookup(blkcg, q); + unsigned long flags; + + if (unlikely(!blkg)) { + spin_lock_irqsave(q->queue_lock, flags); + + blkg = __blkg_lookup_create(blkcg, q); + + spin_unlock_irqrestore(q->queue_lock, flags); + } + + return blkg; +} + static void blkg_destroy(struct blkcg_gq *blkg) { struct blkcg *blkcg = blkg->blkcg; @@ -353,7 +424,7 @@ static void blkg_destroy(struct blkcg_gq *blkg) * Put the reference taken at the time of creation so that when all * queues are gone, group can be destroyed. */ - blkg_put(blkg); + percpu_ref_kill(&blkg->refcnt); } /** @@ -381,29 +452,6 @@ static void blkg_destroy_all(struct request_queue *q) } /* - * A group is RCU protected, but having an rcu lock does not mean that one - * can access all the fields of blkg and assume these are valid. For - * example, don't try to follow throtl_data and request queue links. - * - * Having a reference to blkg under an rcu allows accesses to only values - * local to groups like group stats and group rate limits. - */ -void __blkg_release_rcu(struct rcu_head *rcu_head) -{ - struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head); - - /* release the blkcg and parent blkg refs this blkg has been holding */ - css_put(&blkg->blkcg->css); - if (blkg->parent) - blkg_put(blkg->parent); - - wb_congested_put(blkg->wb_congested); - - blkg_free(blkg); -} -EXPORT_SYMBOL_GPL(__blkg_release_rcu); - -/* * The next function used by blk_queue_for_each_rl(). It's a bit tricky * because the root blkg uses @q->root_rl instead of its own rl. */ @@ -1748,8 +1796,7 @@ void blkcg_maybe_throttle_current(void) blkg = blkg_lookup(blkcg, q); if (!blkg) goto out; - blkg = blkg_try_get(blkg); - if (!blkg) + if (!blkg_tryget(blkg)) goto out; rcu_read_unlock(); diff --git a/block/blk-core.c b/block/blk-core.c index cff0a60ee200..3ed60723e242 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -42,6 +42,7 @@ #include "blk.h" #include "blk-mq.h" #include "blk-mq-sched.h" +#include "blk-pm.h" #include "blk-rq-qos.h" #ifdef CONFIG_DEBUG_FS @@ -421,24 +422,25 @@ void blk_sync_queue(struct request_queue *q) EXPORT_SYMBOL(blk_sync_queue); /** - * blk_set_preempt_only - set QUEUE_FLAG_PREEMPT_ONLY + * blk_set_pm_only - increment pm_only counter * @q: request queue pointer - * - * Returns the previous value of the PREEMPT_ONLY flag - 0 if the flag was not - * set and 1 if the flag was already set. */ -int blk_set_preempt_only(struct request_queue *q) +void blk_set_pm_only(struct request_queue *q) { - return blk_queue_flag_test_and_set(QUEUE_FLAG_PREEMPT_ONLY, q); + atomic_inc(&q->pm_only); } -EXPORT_SYMBOL_GPL(blk_set_preempt_only); +EXPORT_SYMBOL_GPL(blk_set_pm_only); -void blk_clear_preempt_only(struct request_queue *q) +void blk_clear_pm_only(struct request_queue *q) { - blk_queue_flag_clear(QUEUE_FLAG_PREEMPT_ONLY, q); - wake_up_all(&q->mq_freeze_wq); + int pm_only; + + pm_only = atomic_dec_return(&q->pm_only); + WARN_ON_ONCE(pm_only < 0); + if (pm_only == 0) + wake_up_all(&q->mq_freeze_wq); } -EXPORT_SYMBOL_GPL(blk_clear_preempt_only); +EXPORT_SYMBOL_GPL(blk_clear_pm_only); /** * __blk_run_queue_uncond - run a queue whether or not it has been stopped @@ -917,7 +919,7 @@ EXPORT_SYMBOL(blk_alloc_queue); */ int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) { - const bool preempt = flags & BLK_MQ_REQ_PREEMPT; + const bool pm = flags & BLK_MQ_REQ_PREEMPT; while (true) { bool success = false; @@ -925,11 +927,11 @@ int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) rcu_read_lock(); if (percpu_ref_tryget_live(&q->q_usage_counter)) { /* - * The code that sets the PREEMPT_ONLY flag is - * responsible for ensuring that that flag is globally - * visible before the queue is unfrozen. + * The code that increments the pm_only counter is + * responsible for ensuring that that counter is + * globally visible before the queue is unfrozen. */ - if (preempt || !blk_queue_preempt_only(q)) { + if (pm || !blk_queue_pm_only(q)) { success = true; } else { percpu_ref_put(&q->q_usage_counter); @@ -954,7 +956,8 @@ int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) wait_event(q->mq_freeze_wq, (atomic_read(&q->mq_freeze_depth) == 0 && - (preempt || !blk_queue_preempt_only(q))) || + (pm || (blk_pm_request_resume(q), + !blk_queue_pm_only(q)))) || blk_queue_dying(q)); if (blk_queue_dying(q)) return -ENODEV; @@ -1051,8 +1054,7 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id, mutex_init(&q->sysfs_lock); spin_lock_init(&q->__queue_lock); - if (!q->mq_ops) - q->queue_lock = lock ? : &q->__queue_lock; + q->queue_lock = lock ? : &q->__queue_lock; /* * A queue starts its life with bypass turned on to avoid @@ -1160,7 +1162,7 @@ int blk_init_allocated_queue(struct request_queue *q) { WARN_ON_ONCE(q->mq_ops); - q->fq = blk_alloc_flush_queue(q, NUMA_NO_NODE, q->cmd_size); + q->fq = blk_alloc_flush_queue(q, NUMA_NO_NODE, q->cmd_size, GFP_KERNEL); if (!q->fq) return -ENOMEM; @@ -1726,16 +1728,6 @@ void part_round_stats(struct request_queue *q, int cpu, struct hd_struct *part) } EXPORT_SYMBOL_GPL(part_round_stats); -#ifdef CONFIG_PM -static void blk_pm_put_request(struct request *rq) -{ - if (rq->q->dev && !(rq->rq_flags & RQF_PM) && !--rq->q->nr_pending) - pm_runtime_mark_last_busy(rq->q->dev); -} -#else -static inline void blk_pm_put_request(struct request *rq) {} -#endif - void __blk_put_request(struct request_queue *q, struct request *req) { req_flags_t rq_flags = req->rq_flags; @@ -1752,6 +1744,7 @@ void __blk_put_request(struct request_queue *q, struct request *req) blk_req_zone_write_unlock(req); blk_pm_put_request(req); + blk_pm_mark_last_busy(req); elv_completed_request(q, req); @@ -2440,6 +2433,7 @@ blk_qc_t generic_make_request(struct bio *bio) if (q) blk_queue_exit(q); q = bio->bi_disk->queue; + bio_reassociate_blkg(q, bio); flags = 0; if (bio->bi_opf & REQ_NOWAIT) flags = BLK_MQ_REQ_NOWAIT; @@ -2750,30 +2744,6 @@ void blk_account_io_done(struct request *req, u64 now) } } -#ifdef CONFIG_PM -/* - * Don't process normal requests when queue is suspended - * or in the process of suspending/resuming - */ -static bool blk_pm_allow_request(struct request *rq) -{ - switch (rq->q->rpm_status) { - case RPM_RESUMING: - case RPM_SUSPENDING: - return rq->rq_flags & RQF_PM; - case RPM_SUSPENDED: - return false; - default: - return true; - } -} -#else -static bool blk_pm_allow_request(struct request *rq) -{ - return true; -} -#endif - void blk_account_io_start(struct request *rq, bool new_io) { struct hd_struct *part; @@ -2819,11 +2789,14 @@ static struct request *elv_next_request(struct request_queue *q) while (1) { list_for_each_entry(rq, &q->queue_head, queuelist) { - if (blk_pm_allow_request(rq)) - return rq; - - if (rq->rq_flags & RQF_SOFTBARRIER) - break; +#ifdef CONFIG_PM + /* + * If a request gets queued in state RPM_SUSPENDED + * then that's a kernel bug. + */ + WARN_ON_ONCE(q->rpm_status == RPM_SUSPENDED); +#endif + return rq; } /* @@ -3755,191 +3728,6 @@ void blk_finish_plug(struct blk_plug *plug) } EXPORT_SYMBOL(blk_finish_plug); -#ifdef CONFIG_PM -/** - * blk_pm_runtime_init - Block layer runtime PM initialization routine - * @q: the queue of the device - * @dev: the device the queue belongs to - * - * Description: - * Initialize runtime-PM-related fields for @q and start auto suspend for - * @dev. Drivers that want to take advantage of request-based runtime PM - * should call this function after @dev has been initialized, and its - * request queue @q has been allocated, and runtime PM for it can not happen - * yet(either due to disabled/forbidden or its usage_count > 0). In most - * cases, driver should call this function before any I/O has taken place. - * - * This function takes care of setting up using auto suspend for the device, - * the autosuspend delay is set to -1 to make runtime suspend impossible - * until an updated value is either set by user or by driver. Drivers do - * not need to touch other autosuspend settings. - * - * The block layer runtime PM is request based, so only works for drivers - * that use request as their IO unit instead of those directly use bio's. - */ -void blk_pm_runtime_init(struct request_queue *q, struct device *dev) -{ - /* Don't enable runtime PM for blk-mq until it is ready */ - if (q->mq_ops) { - pm_runtime_disable(dev); - return; - } - - q->dev = dev; - q->rpm_status = RPM_ACTIVE; - pm_runtime_set_autosuspend_delay(q->dev, -1); - pm_runtime_use_autosuspend(q->dev); -} -EXPORT_SYMBOL(blk_pm_runtime_init); - -/** - * blk_pre_runtime_suspend - Pre runtime suspend check - * @q: the queue of the device - * - * Description: - * This function will check if runtime suspend is allowed for the device - * by examining if there are any requests pending in the queue. If there - * are requests pending, the device can not be runtime suspended; otherwise, - * the queue's status will be updated to SUSPENDING and the driver can - * proceed to suspend the device. - * - * For the not allowed case, we mark last busy for the device so that - * runtime PM core will try to autosuspend it some time later. - * - * This function should be called near the start of the device's - * runtime_suspend callback. - * - * Return: - * 0 - OK to runtime suspend the device - * -EBUSY - Device should not be runtime suspended - */ -int blk_pre_runtime_suspend(struct request_queue *q) -{ - int ret = 0; - - if (!q->dev) - return ret; - - spin_lock_irq(q->queue_lock); - if (q->nr_pending) { - ret = -EBUSY; - pm_runtime_mark_last_busy(q->dev); - } else { - q->rpm_status = RPM_SUSPENDING; - } - spin_unlock_irq(q->queue_lock); - return ret; -} -EXPORT_SYMBOL(blk_pre_runtime_suspend); - -/** - * blk_post_runtime_suspend - Post runtime suspend processing - * @q: the queue of the device - * @err: return value of the device's runtime_suspend function - * - * Description: - * Update the queue's runtime status according to the return value of the - * device's runtime suspend function and mark last busy for the device so - * that PM core will try to auto suspend the device at a later time. - * - * This function should be called near the end of the device's - * runtime_suspend callback. - */ -void blk_post_runtime_suspend(struct request_queue *q, int err) -{ - if (!q->dev) - return; - - spin_lock_irq(q->queue_lock); - if (!err) { - q->rpm_status = RPM_SUSPENDED; - } else { - q->rpm_status = RPM_ACTIVE; - pm_runtime_mark_last_busy(q->dev); - } - spin_unlock_irq(q->queue_lock); -} -EXPORT_SYMBOL(blk_post_runtime_suspend); - -/** - * blk_pre_runtime_resume - Pre runtime resume processing - * @q: the queue of the device - * - * Description: - * Update the queue's runtime status to RESUMING in preparation for the - * runtime resume of the device. - * - * This function should be called near the start of the device's - * runtime_resume callback. - */ -void blk_pre_runtime_resume(struct request_queue *q) -{ - if (!q->dev) - return; - - spin_lock_irq(q->queue_lock); - q->rpm_status = RPM_RESUMING; - spin_unlock_irq(q->queue_lock); -} -EXPORT_SYMBOL(blk_pre_runtime_resume); - -/** - * blk_post_runtime_resume - Post runtime resume processing - * @q: the queue of the device - * @err: return value of the device's runtime_resume function - * - * Description: - * Update the queue's runtime status according to the return value of the - * device's runtime_resume function. If it is successfully resumed, process - * the requests that are queued into the device's queue when it is resuming - * and then mark last busy and initiate autosuspend for it. - * - * This function should be called near the end of the device's - * runtime_resume callback. - */ -void blk_post_runtime_resume(struct request_queue *q, int err) -{ - if (!q->dev) - return; - - spin_lock_irq(q->queue_lock); - if (!err) { - q->rpm_status = RPM_ACTIVE; - __blk_run_queue(q); - pm_runtime_mark_last_busy(q->dev); - pm_request_autosuspend(q->dev); - } else { - q->rpm_status = RPM_SUSPENDED; - } - spin_unlock_irq(q->queue_lock); -} -EXPORT_SYMBOL(blk_post_runtime_resume); - -/** - * blk_set_runtime_active - Force runtime status of the queue to be active - * @q: the queue of the device - * - * If the device is left runtime suspended during system suspend the resume - * hook typically resumes the device and corrects runtime status - * accordingly. However, that does not affect the queue runtime PM status - * which is still "suspended". This prevents processing requests from the - * queue. - * - * This function can be used in driver's resume hook to correct queue - * runtime PM status and re-enable peeking requests from the queue. It - * should be called before first request is added to the queue. - */ -void blk_set_runtime_active(struct request_queue *q) -{ - spin_lock_irq(q->queue_lock); - q->rpm_status = RPM_ACTIVE; - pm_runtime_mark_last_busy(q->dev); - pm_request_autosuspend(q->dev); - spin_unlock_irq(q->queue_lock); -} -EXPORT_SYMBOL(blk_set_runtime_active); -#endif - int __init blk_dev_init(void) { BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS)); diff --git a/block/blk-flush.c b/block/blk-flush.c index ce41f666de3e..8b44b86779da 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -566,12 +566,12 @@ int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, EXPORT_SYMBOL(blkdev_issue_flush); struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q, - int node, int cmd_size) + int node, int cmd_size, gfp_t flags) { struct blk_flush_queue *fq; int rq_sz = sizeof(struct request); - fq = kzalloc_node(sizeof(*fq), GFP_KERNEL, node); + fq = kzalloc_node(sizeof(*fq), flags, node); if (!fq) goto fail; @@ -579,7 +579,7 @@ struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q, spin_lock_init(&fq->mq_flush_lock); rq_sz = round_up(rq_sz + cmd_size, cache_line_size()); - fq->flush_rq = kzalloc_node(rq_sz, GFP_KERNEL, node); + fq->flush_rq = kzalloc_node(rq_sz, flags, node); if (!fq->flush_rq) goto fail_rq; diff --git a/block/blk-integrity.c b/block/blk-integrity.c index 6121611e1316..d1ab089e0919 100644 --- a/block/blk-integrity.c +++ b/block/blk-integrity.c @@ -49,12 +49,8 @@ int blk_rq_count_integrity_sg(struct request_queue *q, struct bio *bio) bio_for_each_integrity_vec(iv, bio, iter) { if (prev) { - if (!BIOVEC_PHYS_MERGEABLE(&ivprv, &iv)) + if (!biovec_phys_mergeable(q, &ivprv, &iv)) goto new_segment; - - if (!BIOVEC_SEG_BOUNDARY(q, &ivprv, &iv)) - goto new_segment; - if (seg_size + iv.bv_len > queue_max_segment_size(q)) goto new_segment; @@ -95,12 +91,8 @@ int blk_rq_map_integrity_sg(struct request_queue *q, struct bio *bio, bio_for_each_integrity_vec(iv, bio, iter) { if (prev) { - if (!BIOVEC_PHYS_MERGEABLE(&ivprv, &iv)) + if (!biovec_phys_mergeable(q, &ivprv, &iv)) goto new_segment; - - if (!BIOVEC_SEG_BOUNDARY(q, &ivprv, &iv)) - goto new_segment; - if (sg->length + iv.bv_len > queue_max_segment_size(q)) goto new_segment; diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c index 19923f8a029d..35c48d7b8f78 100644 --- a/block/blk-iolatency.c +++ b/block/blk-iolatency.c @@ -115,9 +115,22 @@ struct child_latency_info { atomic_t scale_cookie; }; +struct percentile_stats { + u64 total; + u64 missed; +}; + +struct latency_stat { + union { + struct percentile_stats ps; + struct blk_rq_stat rqs; + }; +}; + struct iolatency_grp { struct blkg_policy_data pd; - struct blk_rq_stat __percpu *stats; + struct latency_stat __percpu *stats; + struct latency_stat cur_stat; struct blk_iolatency *blkiolat; struct rq_depth rq_depth; struct rq_wait rq_wait; @@ -132,6 +145,7 @@ struct iolatency_grp { /* Our current number of IO's for the last summation. */ u64 nr_samples; + bool ssd; struct child_latency_info child_lat; }; @@ -172,6 +186,80 @@ static inline struct blkcg_gq *lat_to_blkg(struct iolatency_grp *iolat) return pd_to_blkg(&iolat->pd); } +static inline void latency_stat_init(struct iolatency_grp *iolat, + struct latency_stat *stat) +{ + if (iolat->ssd) { + stat->ps.total = 0; + stat->ps.missed = 0; + } else + blk_rq_stat_init(&stat->rqs); +} + +static inline void latency_stat_sum(struct iolatency_grp *iolat, + struct latency_stat *sum, + struct latency_stat *stat) +{ + if (iolat->ssd) { + sum->ps.total += stat->ps.total; + sum->ps.missed += stat->ps.missed; + } else + blk_rq_stat_sum(&sum->rqs, &stat->rqs); +} + +static inline void latency_stat_record_time(struct iolatency_grp *iolat, + u64 req_time) +{ + struct latency_stat *stat = get_cpu_ptr(iolat->stats); + if (iolat->ssd) { + if (req_time >= iolat->min_lat_nsec) + stat->ps.missed++; + stat->ps.total++; + } else + blk_rq_stat_add(&stat->rqs, req_time); + put_cpu_ptr(stat); +} + +static inline bool latency_sum_ok(struct iolatency_grp *iolat, + struct latency_stat *stat) +{ + if (iolat->ssd) { + u64 thresh = div64_u64(stat->ps.total, 10); + thresh = max(thresh, 1ULL); + return stat->ps.missed < thresh; + } + return stat->rqs.mean <= iolat->min_lat_nsec; +} + +static inline u64 latency_stat_samples(struct iolatency_grp *iolat, + struct latency_stat *stat) +{ + if (iolat->ssd) + return stat->ps.total; + return stat->rqs.nr_samples; +} + +static inline void iolat_update_total_lat_avg(struct iolatency_grp *iolat, + struct latency_stat *stat) +{ + int exp_idx; + + if (iolat->ssd) + return; + + /* + * CALC_LOAD takes in a number stored in fixed point representation. + * Because we are using this for IO time in ns, the values stored + * are significantly larger than the FIXED_1 denominator (2048). + * Therefore, rounding errors in the calculation are negligible and + * can be ignored. + */ + exp_idx = min_t(int, BLKIOLATENCY_NR_EXP_FACTORS - 1, + div64_u64(iolat->cur_win_nsec, + BLKIOLATENCY_EXP_BUCKET_SIZE)); + CALC_LOAD(iolat->lat_avg, iolatency_exp_factors[exp_idx], stat->rqs.mean); +} + static inline bool iolatency_may_queue(struct iolatency_grp *iolat, wait_queue_entry_t *wait, bool first_block) @@ -255,7 +343,7 @@ static void scale_cookie_change(struct blk_iolatency *blkiolat, struct child_latency_info *lat_info, bool up) { - unsigned long qd = blk_queue_depth(blkiolat->rqos.q); + unsigned long qd = blkiolat->rqos.q->nr_requests; unsigned long scale = scale_amount(qd, up); unsigned long old = atomic_read(&lat_info->scale_cookie); unsigned long max_scale = qd << 1; @@ -295,10 +383,9 @@ static void scale_cookie_change(struct blk_iolatency *blkiolat, */ static void scale_change(struct iolatency_grp *iolat, bool up) { - unsigned long qd = blk_queue_depth(iolat->blkiolat->rqos.q); + unsigned long qd = iolat->blkiolat->rqos.q->nr_requests; unsigned long scale = scale_amount(qd, up); unsigned long old = iolat->rq_depth.max_depth; - bool changed = false; if (old > qd) old = qd; @@ -308,15 +395,13 @@ static void scale_change(struct iolatency_grp *iolat, bool up) return; if (old < qd) { - changed = true; old += scale; old = min(old, qd); iolat->rq_depth.max_depth = old; wake_up_all(&iolat->rq_wait.wait); } - } else if (old > 1) { + } else { old >>= 1; - changed = true; iolat->rq_depth.max_depth = max(old, 1UL); } } @@ -369,7 +454,7 @@ static void check_scale_change(struct iolatency_grp *iolat) * scale down event. */ samples_thresh = lat_info->nr_samples * 5; - samples_thresh = div64_u64(samples_thresh, 100); + samples_thresh = max(1ULL, div64_u64(samples_thresh, 100)); if (iolat->nr_samples <= samples_thresh) return; } @@ -395,34 +480,12 @@ static void blkcg_iolatency_throttle(struct rq_qos *rqos, struct bio *bio, spinlock_t *lock) { struct blk_iolatency *blkiolat = BLKIOLATENCY(rqos); - struct blkcg *blkcg; - struct blkcg_gq *blkg; - struct request_queue *q = rqos->q; + struct blkcg_gq *blkg = bio->bi_blkg; bool issue_as_root = bio_issue_as_root_blkg(bio); if (!blk_iolatency_enabled(blkiolat)) return; - rcu_read_lock(); - blkcg = bio_blkcg(bio); - bio_associate_blkcg(bio, &blkcg->css); - blkg = blkg_lookup(blkcg, q); - if (unlikely(!blkg)) { - if (!lock) - spin_lock_irq(q->queue_lock); - blkg = blkg_lookup_create(blkcg, q); - if (IS_ERR(blkg)) - blkg = NULL; - if (!lock) - spin_unlock_irq(q->queue_lock); - } - if (!blkg) - goto out; - - bio_issue_init(&bio->bi_issue, bio_sectors(bio)); - bio_associate_blkg(bio, blkg); -out: - rcu_read_unlock(); while (blkg && blkg->parent) { struct iolatency_grp *iolat = blkg_to_lat(blkg); if (!iolat) { @@ -443,7 +506,6 @@ static void iolatency_record_time(struct iolatency_grp *iolat, struct bio_issue *issue, u64 now, bool issue_as_root) { - struct blk_rq_stat *rq_stat; u64 start = bio_issue_time(issue); u64 req_time; @@ -469,9 +531,7 @@ static void iolatency_record_time(struct iolatency_grp *iolat, return; } - rq_stat = get_cpu_ptr(iolat->stats); - blk_rq_stat_add(rq_stat, req_time); - put_cpu_ptr(rq_stat); + latency_stat_record_time(iolat, req_time); } #define BLKIOLATENCY_MIN_ADJUST_TIME (500 * NSEC_PER_MSEC) @@ -482,17 +542,17 @@ static void iolatency_check_latencies(struct iolatency_grp *iolat, u64 now) struct blkcg_gq *blkg = lat_to_blkg(iolat); struct iolatency_grp *parent; struct child_latency_info *lat_info; - struct blk_rq_stat stat; + struct latency_stat stat; unsigned long flags; - int cpu, exp_idx; + int cpu; - blk_rq_stat_init(&stat); + latency_stat_init(iolat, &stat); preempt_disable(); for_each_online_cpu(cpu) { - struct blk_rq_stat *s; + struct latency_stat *s; s = per_cpu_ptr(iolat->stats, cpu); - blk_rq_stat_sum(&stat, s); - blk_rq_stat_init(s); + latency_stat_sum(iolat, &stat, s); + latency_stat_init(iolat, s); } preempt_enable(); @@ -502,41 +562,36 @@ static void iolatency_check_latencies(struct iolatency_grp *iolat, u64 now) lat_info = &parent->child_lat; - /* - * CALC_LOAD takes in a number stored in fixed point representation. - * Because we are using this for IO time in ns, the values stored - * are significantly larger than the FIXED_1 denominator (2048). - * Therefore, rounding errors in the calculation are negligible and - * can be ignored. - */ - exp_idx = min_t(int, BLKIOLATENCY_NR_EXP_FACTORS - 1, - div64_u64(iolat->cur_win_nsec, - BLKIOLATENCY_EXP_BUCKET_SIZE)); - CALC_LOAD(iolat->lat_avg, iolatency_exp_factors[exp_idx], stat.mean); + iolat_update_total_lat_avg(iolat, &stat); /* Everything is ok and we don't need to adjust the scale. */ - if (stat.mean <= iolat->min_lat_nsec && + if (latency_sum_ok(iolat, &stat) && atomic_read(&lat_info->scale_cookie) == DEFAULT_SCALE_COOKIE) return; /* Somebody beat us to the punch, just bail. */ spin_lock_irqsave(&lat_info->lock, flags); + + latency_stat_sum(iolat, &iolat->cur_stat, &stat); lat_info->nr_samples -= iolat->nr_samples; - lat_info->nr_samples += stat.nr_samples; - iolat->nr_samples = stat.nr_samples; + lat_info->nr_samples += latency_stat_samples(iolat, &iolat->cur_stat); + iolat->nr_samples = latency_stat_samples(iolat, &iolat->cur_stat); if ((lat_info->last_scale_event >= now || - now - lat_info->last_scale_event < BLKIOLATENCY_MIN_ADJUST_TIME) && - lat_info->scale_lat <= iolat->min_lat_nsec) + now - lat_info->last_scale_event < BLKIOLATENCY_MIN_ADJUST_TIME)) goto out; - if (stat.mean <= iolat->min_lat_nsec && - stat.nr_samples >= BLKIOLATENCY_MIN_GOOD_SAMPLES) { + if (latency_sum_ok(iolat, &iolat->cur_stat) && + latency_sum_ok(iolat, &stat)) { + if (latency_stat_samples(iolat, &iolat->cur_stat) < + BLKIOLATENCY_MIN_GOOD_SAMPLES) + goto out; if (lat_info->scale_grp == iolat) { lat_info->last_scale_event = now; scale_cookie_change(iolat->blkiolat, lat_info, true); } - } else if (stat.mean > iolat->min_lat_nsec) { + } else if (lat_info->scale_lat == 0 || + lat_info->scale_lat >= iolat->min_lat_nsec) { lat_info->last_scale_event = now; if (!lat_info->scale_grp || lat_info->scale_lat > iolat->min_lat_nsec) { @@ -545,6 +600,7 @@ static void iolatency_check_latencies(struct iolatency_grp *iolat, u64 now) } scale_cookie_change(iolat->blkiolat, lat_info, false); } + latency_stat_init(iolat, &iolat->cur_stat); out: spin_unlock_irqrestore(&lat_info->lock, flags); } @@ -650,7 +706,7 @@ static void blkiolatency_timer_fn(struct timer_list *t) * We could be exiting, don't access the pd unless we have a * ref on the blkg. */ - if (!blkg_try_get(blkg)) + if (!blkg_tryget(blkg)) continue; iolat = blkg_to_lat(blkg); @@ -761,7 +817,6 @@ static ssize_t iolatency_set_limit(struct kernfs_open_file *of, char *buf, { struct blkcg *blkcg = css_to_blkcg(of_css(of)); struct blkcg_gq *blkg; - struct blk_iolatency *blkiolat; struct blkg_conf_ctx ctx; struct iolatency_grp *iolat; char *p, *tok; @@ -774,7 +829,6 @@ static ssize_t iolatency_set_limit(struct kernfs_open_file *of, char *buf, return ret; iolat = blkg_to_lat(ctx.blkg); - blkiolat = iolat->blkiolat; p = ctx.body; ret = -EINVAL; @@ -835,13 +889,43 @@ static int iolatency_print_limit(struct seq_file *sf, void *v) return 0; } +static size_t iolatency_ssd_stat(struct iolatency_grp *iolat, char *buf, + size_t size) +{ + struct latency_stat stat; + int cpu; + + latency_stat_init(iolat, &stat); + preempt_disable(); + for_each_online_cpu(cpu) { + struct latency_stat *s; + s = per_cpu_ptr(iolat->stats, cpu); + latency_stat_sum(iolat, &stat, s); + } + preempt_enable(); + + if (iolat->rq_depth.max_depth == UINT_MAX) + return scnprintf(buf, size, " missed=%llu total=%llu depth=max", + (unsigned long long)stat.ps.missed, + (unsigned long long)stat.ps.total); + return scnprintf(buf, size, " missed=%llu total=%llu depth=%u", + (unsigned long long)stat.ps.missed, + (unsigned long long)stat.ps.total, + iolat->rq_depth.max_depth); +} + static size_t iolatency_pd_stat(struct blkg_policy_data *pd, char *buf, size_t size) { struct iolatency_grp *iolat = pd_to_lat(pd); - unsigned long long avg_lat = div64_u64(iolat->lat_avg, NSEC_PER_USEC); - unsigned long long cur_win = div64_u64(iolat->cur_win_nsec, NSEC_PER_MSEC); + unsigned long long avg_lat; + unsigned long long cur_win; + + if (iolat->ssd) + return iolatency_ssd_stat(iolat, buf, size); + avg_lat = div64_u64(iolat->lat_avg, NSEC_PER_USEC); + cur_win = div64_u64(iolat->cur_win_nsec, NSEC_PER_MSEC); if (iolat->rq_depth.max_depth == UINT_MAX) return scnprintf(buf, size, " depth=max avg_lat=%llu win=%llu", avg_lat, cur_win); @@ -858,8 +942,8 @@ static struct blkg_policy_data *iolatency_pd_alloc(gfp_t gfp, int node) iolat = kzalloc_node(sizeof(*iolat), gfp, node); if (!iolat) return NULL; - iolat->stats = __alloc_percpu_gfp(sizeof(struct blk_rq_stat), - __alignof__(struct blk_rq_stat), gfp); + iolat->stats = __alloc_percpu_gfp(sizeof(struct latency_stat), + __alignof__(struct latency_stat), gfp); if (!iolat->stats) { kfree(iolat); return NULL; @@ -876,15 +960,21 @@ static void iolatency_pd_init(struct blkg_policy_data *pd) u64 now = ktime_to_ns(ktime_get()); int cpu; + if (blk_queue_nonrot(blkg->q)) + iolat->ssd = true; + else + iolat->ssd = false; + for_each_possible_cpu(cpu) { - struct blk_rq_stat *stat; + struct latency_stat *stat; stat = per_cpu_ptr(iolat->stats, cpu); - blk_rq_stat_init(stat); + latency_stat_init(iolat, stat); } + latency_stat_init(iolat, &iolat->cur_stat); rq_wait_init(&iolat->rq_wait); spin_lock_init(&iolat->child_lat.lock); - iolat->rq_depth.queue_depth = blk_queue_depth(blkg->q); + iolat->rq_depth.queue_depth = blkg->q->nr_requests; iolat->rq_depth.max_depth = UINT_MAX; iolat->rq_depth.default_depth = iolat->rq_depth.queue_depth; iolat->blkiolat = blkiolat; diff --git a/block/blk-lib.c b/block/blk-lib.c index d1b9dd03da25..bbd44666f2b5 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c @@ -29,9 +29,7 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, { struct request_queue *q = bdev_get_queue(bdev); struct bio *bio = *biop; - unsigned int granularity; unsigned int op; - int alignment; sector_t bs_mask; if (!q) @@ -54,38 +52,16 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, if ((sector | nr_sects) & bs_mask) return -EINVAL; - /* Zero-sector (unknown) and one-sector granularities are the same. */ - granularity = max(q->limits.discard_granularity >> 9, 1U); - alignment = (bdev_discard_alignment(bdev) >> 9) % granularity; - while (nr_sects) { - unsigned int req_sects; - sector_t end_sect, tmp; + unsigned int req_sects = nr_sects; + sector_t end_sect; - /* - * Issue in chunks of the user defined max discard setting, - * ensuring that bi_size doesn't overflow - */ - req_sects = min_t(sector_t, nr_sects, - q->limits.max_discard_sectors); if (!req_sects) goto fail; if (req_sects > UINT_MAX >> 9) req_sects = UINT_MAX >> 9; - /* - * If splitting a request, and the next starting sector would be - * misaligned, stop the discard at the previous aligned sector. - */ end_sect = sector + req_sects; - tmp = end_sect; - if (req_sects < nr_sects && - sector_div(tmp, granularity) != alignment) { - end_sect = end_sect - alignment; - sector_div(end_sect, granularity); - end_sect = end_sect * granularity + alignment; - req_sects = end_sect - sector; - } bio = next_bio(bio, 0, gfp_mask); bio->bi_iter.bi_sector = sector; diff --git a/block/blk-merge.c b/block/blk-merge.c index aaec38cc37b8..42a46744c11b 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -12,6 +12,69 @@ #include "blk.h" +/* + * Check if the two bvecs from two bios can be merged to one segment. If yes, + * no need to check gap between the two bios since the 1st bio and the 1st bvec + * in the 2nd bio can be handled in one segment. + */ +static inline bool bios_segs_mergeable(struct request_queue *q, + struct bio *prev, struct bio_vec *prev_last_bv, + struct bio_vec *next_first_bv) +{ + if (!biovec_phys_mergeable(q, prev_last_bv, next_first_bv)) + return false; + if (prev->bi_seg_back_size + next_first_bv->bv_len > + queue_max_segment_size(q)) + return false; + return true; +} + +static inline bool bio_will_gap(struct request_queue *q, + struct request *prev_rq, struct bio *prev, struct bio *next) +{ + struct bio_vec pb, nb; + + if (!bio_has_data(prev) || !queue_virt_boundary(q)) + return false; + + /* + * Don't merge if the 1st bio starts with non-zero offset, otherwise it + * is quite difficult to respect the sg gap limit. We work hard to + * merge a huge number of small single bios in case of mkfs. + */ + if (prev_rq) + bio_get_first_bvec(prev_rq->bio, &pb); + else + bio_get_first_bvec(prev, &pb); + if (pb.bv_offset) + return true; + + /* + * We don't need to worry about the situation that the merged segment + * ends in unaligned virt boundary: + * + * - if 'pb' ends aligned, the merged segment ends aligned + * - if 'pb' ends unaligned, the next bio must include + * one single bvec of 'nb', otherwise the 'nb' can't + * merge with 'pb' + */ + bio_get_last_bvec(prev, &pb); + bio_get_first_bvec(next, &nb); + if (bios_segs_mergeable(q, prev, &pb, &nb)) + return false; + return __bvec_gap_to_prev(q, &pb, nb.bv_offset); +} + +static inline bool req_gap_back_merge(struct request *req, struct bio *bio) +{ + return bio_will_gap(req->q, req, req->biotail, bio); +} + +static inline bool req_gap_front_merge(struct request *req, struct bio *bio) +{ + return bio_will_gap(req->q, NULL, bio, req->bio); +} + static struct bio *blk_bio_discard_split(struct request_queue *q, struct bio *bio, struct bio_set *bs, @@ -134,9 +197,7 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, if (bvprvp && blk_queue_cluster(q)) { if (seg_size + bv.bv_len > queue_max_segment_size(q)) goto new_segment; - if (!BIOVEC_PHYS_MERGEABLE(bvprvp, &bv)) - goto new_segment; - if (!BIOVEC_SEG_BOUNDARY(q, bvprvp, &bv)) + if (!biovec_phys_mergeable(q, bvprvp, &bv)) goto new_segment; seg_size += bv.bv_len; @@ -267,9 +328,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, if (seg_size + bv.bv_len > queue_max_segment_size(q)) goto new_segment; - if (!BIOVEC_PHYS_MERGEABLE(&bvprv, &bv)) - goto new_segment; - if (!BIOVEC_SEG_BOUNDARY(q, &bvprv, &bv)) + if (!biovec_phys_mergeable(q, &bvprv, &bv)) goto new_segment; seg_size += bv.bv_len; @@ -349,17 +408,7 @@ static int blk_phys_contig_segment(struct request_queue *q, struct bio *bio, bio_get_last_bvec(bio, &end_bv); bio_get_first_bvec(nxt, &nxt_bv); - if (!BIOVEC_PHYS_MERGEABLE(&end_bv, &nxt_bv)) - return 0; - - /* - * bio and nxt are contiguous in memory; check if the queue allows - * these two to be merged into one - */ - if (BIOVEC_SEG_BOUNDARY(q, &end_bv, &nxt_bv)) - return 1; - - return 0; + return biovec_phys_mergeable(q, &end_bv, &nxt_bv); } static inline void @@ -373,10 +422,7 @@ __blk_segment_map_sg(struct request_queue *q, struct bio_vec *bvec, if (*sg && *cluster) { if ((*sg)->length + nbytes > queue_max_segment_size(q)) goto new_segment; - - if (!BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) - goto new_segment; - if (!BIOVEC_SEG_BOUNDARY(q, bvprv, bvec)) + if (!biovec_phys_mergeable(q, bvprv, bvec)) goto new_segment; (*sg)->length += nbytes; diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index cb1e6cf7ac48..41b86f50d126 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -102,6 +102,14 @@ static int blk_flags_show(struct seq_file *m, const unsigned long flags, return 0; } +static int queue_pm_only_show(void *data, struct seq_file *m) +{ + struct request_queue *q = data; + + seq_printf(m, "%d\n", atomic_read(&q->pm_only)); + return 0; +} + #define QUEUE_FLAG_NAME(name) [QUEUE_FLAG_##name] = #name static const char *const blk_queue_flag_name[] = { QUEUE_FLAG_NAME(QUEUED), @@ -132,7 +140,6 @@ static const char *const blk_queue_flag_name[] = { QUEUE_FLAG_NAME(REGISTERED), QUEUE_FLAG_NAME(SCSI_PASSTHROUGH), QUEUE_FLAG_NAME(QUIESCED), - QUEUE_FLAG_NAME(PREEMPT_ONLY), }; #undef QUEUE_FLAG_NAME @@ -209,6 +216,7 @@ static ssize_t queue_write_hint_store(void *data, const char __user *buf, static const struct blk_mq_debugfs_attr blk_mq_debugfs_queue_attrs[] = { { "poll_stat", 0400, queue_poll_stat_show }, { "requeue_list", 0400, .seq_ops = &queue_requeue_list_seq_ops }, + { "pm_only", 0600, queue_pm_only_show, NULL }, { "state", 0600, queue_state_show, queue_state_write }, { "write_hints", 0600, queue_write_hint_show, queue_write_hint_store }, { "zone_wlock", 0400, queue_zone_wlock_show, NULL }, @@ -423,8 +431,7 @@ static void hctx_show_busy_rq(struct request *rq, void *data, bool reserved) { const struct show_busy_params *params = data; - if (blk_mq_map_queue(rq->q, rq->mq_ctx->cpu) == params->hctx && - blk_mq_rq_state(rq) != MQ_RQ_IDLE) + if (blk_mq_map_queue(rq->q, rq->mq_ctx->cpu) == params->hctx) __blk_mq_debugfs_rq_show(params->m, list_entry_rq(&rq->queuelist)); } diff --git a/block/blk-mq-sched.h b/block/blk-mq-sched.h index 4e028ee42430..8a9544203173 100644 --- a/block/blk-mq-sched.h +++ b/block/blk-mq-sched.h @@ -49,12 +49,12 @@ blk_mq_sched_allow_merge(struct request_queue *q, struct request *rq, return true; } -static inline void blk_mq_sched_completed_request(struct request *rq) +static inline void blk_mq_sched_completed_request(struct request *rq, u64 now) { struct elevator_queue *e = rq->q->elevator; if (e && e->type->ops.mq.completed_request) - e->type->ops.mq.completed_request(rq); + e->type->ops.mq.completed_request(rq, now); } static inline void blk_mq_sched_started_request(struct request *rq) diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 41317c50a446..cfda95b85d34 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -232,13 +232,26 @@ static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) /* * We can hit rq == NULL here, because the tagging functions - * test and set the bit before assining ->rqs[]. + * test and set the bit before assigning ->rqs[]. */ if (rq && rq->q == hctx->queue) iter_data->fn(hctx, rq, iter_data->data, reserved); return true; } +/** + * bt_for_each - iterate over the requests associated with a hardware queue + * @hctx: Hardware queue to examine. + * @bt: sbitmap to examine. This is either the breserved_tags member + * or the bitmap_tags member of struct blk_mq_tags. + * @fn: Pointer to the function that will be called for each request + * associated with @hctx that has been assigned a driver tag. + * @fn will be called as follows: @fn(@hctx, rq, @data, @reserved) + * where rq is a pointer to a request. + * @data: Will be passed as third argument to @fn. + * @reserved: Indicates whether @bt is the breserved_tags member or the + * bitmap_tags member of struct blk_mq_tags. + */ static void bt_for_each(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt, busy_iter_fn *fn, void *data, bool reserved) { @@ -280,6 +293,18 @@ static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) return true; } +/** + * bt_tags_for_each - iterate over the requests in a tag map + * @tags: Tag map to iterate over. + * @bt: sbitmap to examine. This is either the breserved_tags member + * or the bitmap_tags member of struct blk_mq_tags. + * @fn: Pointer to the function that will be called for each started + * request. @fn will be called as follows: @fn(rq, @data, + * @reserved) where rq is a pointer to a request. + * @data: Will be passed as second argument to @fn. + * @reserved: Indicates whether @bt is the breserved_tags member or the + * bitmap_tags member of struct blk_mq_tags. + */ static void bt_tags_for_each(struct blk_mq_tags *tags, struct sbitmap_queue *bt, busy_tag_iter_fn *fn, void *data, bool reserved) { @@ -294,6 +319,15 @@ static void bt_tags_for_each(struct blk_mq_tags *tags, struct sbitmap_queue *bt, sbitmap_for_each_set(&bt->sb, bt_tags_iter, &iter_data); } +/** + * blk_mq_all_tag_busy_iter - iterate over all started requests in a tag map + * @tags: Tag map to iterate over. + * @fn: Pointer to the function that will be called for each started + * request. @fn will be called as follows: @fn(rq, @priv, + * reserved) where rq is a pointer to a request. 'reserved' + * indicates whether or not @rq is a reserved request. + * @priv: Will be passed as second argument to @fn. + */ static void blk_mq_all_tag_busy_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn, void *priv) { @@ -302,6 +336,15 @@ static void blk_mq_all_tag_busy_iter(struct blk_mq_tags *tags, bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, false); } +/** + * blk_mq_tagset_busy_iter - iterate over all started requests in a tag set + * @tagset: Tag set to iterate over. + * @fn: Pointer to the function that will be called for each started + * request. @fn will be called as follows: @fn(rq, @priv, + * reserved) where rq is a pointer to a request. 'reserved' + * indicates whether or not @rq is a reserved request. + * @priv: Will be passed as second argument to @fn. + */ void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, busy_tag_iter_fn *fn, void *priv) { @@ -314,6 +357,20 @@ void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, } EXPORT_SYMBOL(blk_mq_tagset_busy_iter); +/** + * blk_mq_queue_tag_busy_iter - iterate over all requests with a driver tag + * @q: Request queue to examine. + * @fn: Pointer to the function that will be called for each request + * on @q. @fn will be called as follows: @fn(hctx, rq, @priv, + * reserved) where rq is a pointer to a request and hctx points + * to the hardware queue associated with the request. 'reserved' + * indicates whether or not @rq is a reserved request. + * @priv: Will be passed as third argument to @fn. + * + * Note: if @q->tag_set is shared with other request queues then @fn will be + * called for all requests on all queues that share that tag set and not only + * for requests associated with @q. + */ void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn, void *priv) { @@ -321,9 +378,11 @@ void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn, int i; /* - * __blk_mq_update_nr_hw_queues will update the nr_hw_queues and - * queue_hw_ctx after freeze the queue, so we use q_usage_counter - * to avoid race with it. + * __blk_mq_update_nr_hw_queues() updates nr_hw_queues and queue_hw_ctx + * while the queue is frozen. So we can use q_usage_counter to avoid + * racing with it. __blk_mq_update_nr_hw_queues() uses + * synchronize_rcu() to ensure this function left the critical section + * below. */ if (!percpu_ref_tryget(&q->q_usage_counter)) return; @@ -332,7 +391,7 @@ void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn, struct blk_mq_tags *tags = hctx->tags; /* - * If not software queues are currently mapped to this + * If no software queues are currently mapped to this * hardware queue, there's nothing to check */ if (!blk_mq_hw_queue_mapped(hctx)) diff --git a/block/blk-mq.c b/block/blk-mq.c index e3c39ea8e17b..dcf10e39995a 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -33,6 +33,7 @@ #include "blk-mq.h" #include "blk-mq-debugfs.h" #include "blk-mq-tag.h" +#include "blk-pm.h" #include "blk-stat.h" #include "blk-mq-sched.h" #include "blk-rq-qos.h" @@ -198,7 +199,7 @@ void blk_mq_unfreeze_queue(struct request_queue *q) freeze_depth = atomic_dec_return(&q->mq_freeze_depth); WARN_ON_ONCE(freeze_depth < 0); if (!freeze_depth) { - percpu_ref_reinit(&q->q_usage_counter); + percpu_ref_resurrect(&q->q_usage_counter); wake_up_all(&q->mq_freeze_wq); } } @@ -475,6 +476,7 @@ static void __blk_mq_free_request(struct request *rq) struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu); const int sched_tag = rq->internal_tag; + blk_pm_mark_last_busy(rq); if (rq->tag != -1) blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag); if (sched_tag != -1) @@ -526,6 +528,9 @@ inline void __blk_mq_end_request(struct request *rq, blk_status_t error) blk_stat_add(rq, now); } + if (rq->internal_tag != -1) + blk_mq_sched_completed_request(rq, now); + blk_account_io_done(rq, now); if (rq->end_io) { @@ -562,8 +567,20 @@ static void __blk_mq_complete_request(struct request *rq) if (!blk_mq_mark_complete(rq)) return; - if (rq->internal_tag != -1) - blk_mq_sched_completed_request(rq); + + /* + * Most of single queue controllers, there is only one irq vector + * for handling IO completion, and the only irq's affinity is set + * as all possible CPUs. On most of ARCHs, this affinity means the + * irq is handled on one specific CPU. + * + * So complete IO reqeust in softirq context in case of single queue + * for not degrading IO performance by irqsoff latency. + */ + if (rq->q->nr_hw_queues == 1) { + __blk_complete_request(rq); + return; + } if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) { rq->q->softirq_done_fn(rq); @@ -2137,8 +2154,6 @@ static void blk_mq_exit_hctx(struct request_queue *q, struct blk_mq_tag_set *set, struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) { - blk_mq_debugfs_unregister_hctx(hctx); - if (blk_mq_hw_queue_mapped(hctx)) blk_mq_tag_idle(hctx); @@ -2165,6 +2180,7 @@ static void blk_mq_exit_hw_queues(struct request_queue *q, queue_for_each_hw_ctx(q, hctx, i) { if (i == nr_queue) break; + blk_mq_debugfs_unregister_hctx(hctx); blk_mq_exit_hctx(q, set, hctx, i); } } @@ -2194,12 +2210,12 @@ static int blk_mq_init_hctx(struct request_queue *q, * runtime */ hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *), - GFP_KERNEL, node); + GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, node); if (!hctx->ctxs) goto unregister_cpu_notifier; - if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL, - node)) + if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), + GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, node)) goto free_ctxs; hctx->nr_ctx = 0; @@ -2212,7 +2228,8 @@ static int blk_mq_init_hctx(struct request_queue *q, set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) goto free_bitmap; - hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size); + hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size, + GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY); if (!hctx->fq) goto exit_hctx; @@ -2222,8 +2239,6 @@ static int blk_mq_init_hctx(struct request_queue *q, if (hctx->flags & BLK_MQ_F_BLOCKING) init_srcu_struct(hctx->srcu); - blk_mq_debugfs_register_hctx(q, hctx); - return 0; free_fq: @@ -2492,6 +2507,39 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) } EXPORT_SYMBOL(blk_mq_init_queue); +/* + * Helper for setting up a queue with mq ops, given queue depth, and + * the passed in mq ops flags. + */ +struct request_queue *blk_mq_init_sq_queue(struct blk_mq_tag_set *set, + const struct blk_mq_ops *ops, + unsigned int queue_depth, + unsigned int set_flags) +{ + struct request_queue *q; + int ret; + + memset(set, 0, sizeof(*set)); + set->ops = ops; + set->nr_hw_queues = 1; + set->queue_depth = queue_depth; + set->numa_node = NUMA_NO_NODE; + set->flags = set_flags; + + ret = blk_mq_alloc_tag_set(set); + if (ret) + return ERR_PTR(ret); + + q = blk_mq_init_queue(set); + if (IS_ERR(q)) { + blk_mq_free_tag_set(set); + return q; + } + + return q; +} +EXPORT_SYMBOL(blk_mq_init_sq_queue); + static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set) { int hw_ctx_size = sizeof(struct blk_mq_hw_ctx); @@ -2506,48 +2554,90 @@ static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set) return hw_ctx_size; } +static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx( + struct blk_mq_tag_set *set, struct request_queue *q, + int hctx_idx, int node) +{ + struct blk_mq_hw_ctx *hctx; + + hctx = kzalloc_node(blk_mq_hw_ctx_size(set), + GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, + node); + if (!hctx) + return NULL; + + if (!zalloc_cpumask_var_node(&hctx->cpumask, + GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, + node)) { + kfree(hctx); + return NULL; + } + + atomic_set(&hctx->nr_active, 0); + hctx->numa_node = node; + hctx->queue_num = hctx_idx; + + if (blk_mq_init_hctx(q, set, hctx, hctx_idx)) { + free_cpumask_var(hctx->cpumask); + kfree(hctx); + return NULL; + } + blk_mq_hctx_kobj_init(hctx); + + return hctx; +} + static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, struct request_queue *q) { - int i, j; + int i, j, end; struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx; - blk_mq_sysfs_unregister(q); - /* protect against switching io scheduler */ mutex_lock(&q->sysfs_lock); for (i = 0; i < set->nr_hw_queues; i++) { int node; - - if (hctxs[i]) - continue; + struct blk_mq_hw_ctx *hctx; node = blk_mq_hw_queue_to_node(q->mq_map, i); - hctxs[i] = kzalloc_node(blk_mq_hw_ctx_size(set), - GFP_KERNEL, node); - if (!hctxs[i]) - break; - - if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL, - node)) { - kfree(hctxs[i]); - hctxs[i] = NULL; - break; - } - - atomic_set(&hctxs[i]->nr_active, 0); - hctxs[i]->numa_node = node; - hctxs[i]->queue_num = i; + /* + * If the hw queue has been mapped to another numa node, + * we need to realloc the hctx. If allocation fails, fallback + * to use the previous one. + */ + if (hctxs[i] && (hctxs[i]->numa_node == node)) + continue; - if (blk_mq_init_hctx(q, set, hctxs[i], i)) { - free_cpumask_var(hctxs[i]->cpumask); - kfree(hctxs[i]); - hctxs[i] = NULL; - break; + hctx = blk_mq_alloc_and_init_hctx(set, q, i, node); + if (hctx) { + if (hctxs[i]) { + blk_mq_exit_hctx(q, set, hctxs[i], i); + kobject_put(&hctxs[i]->kobj); + } + hctxs[i] = hctx; + } else { + if (hctxs[i]) + pr_warn("Allocate new hctx on node %d fails,\ + fallback to previous one on node %d\n", + node, hctxs[i]->numa_node); + else + break; } - blk_mq_hctx_kobj_init(hctxs[i]); } - for (j = i; j < q->nr_hw_queues; j++) { + /* + * Increasing nr_hw_queues fails. Free the newly allocated + * hctxs and keep the previous q->nr_hw_queues. + */ + if (i != set->nr_hw_queues) { + j = q->nr_hw_queues; + end = i; + } else { + j = i; + end = q->nr_hw_queues; + q->nr_hw_queues = set->nr_hw_queues; + } + + for (; j < end; j++) { struct blk_mq_hw_ctx *hctx = hctxs[j]; if (hctx) { @@ -2559,9 +2649,7 @@ static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, } } - q->nr_hw_queues = i; mutex_unlock(&q->sysfs_lock); - blk_mq_sysfs_register(q); } struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, @@ -2659,25 +2747,6 @@ void blk_mq_free_queue(struct request_queue *q) blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); } -/* Basically redo blk_mq_init_queue with queue frozen */ -static void blk_mq_queue_reinit(struct request_queue *q) -{ - WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth)); - - blk_mq_debugfs_unregister_hctxs(q); - blk_mq_sysfs_unregister(q); - - /* - * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe - * we should change hctx numa_node according to the new topology (this - * involves freeing and re-allocating memory, worth doing?) - */ - blk_mq_map_swqueue(q); - - blk_mq_sysfs_register(q); - blk_mq_debugfs_register_hctxs(q); -} - static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) { int i; @@ -2964,6 +3033,7 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, { struct request_queue *q; LIST_HEAD(head); + int prev_nr_hw_queues; lockdep_assert_held(&set->tag_list_lock); @@ -2987,11 +3057,30 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, if (!blk_mq_elv_switch_none(&head, q)) goto switch_back; + list_for_each_entry(q, &set->tag_list, tag_set_list) { + blk_mq_debugfs_unregister_hctxs(q); + blk_mq_sysfs_unregister(q); + } + + prev_nr_hw_queues = set->nr_hw_queues; set->nr_hw_queues = nr_hw_queues; blk_mq_update_queue_map(set); +fallback: list_for_each_entry(q, &set->tag_list, tag_set_list) { blk_mq_realloc_hw_ctxs(set, q); - blk_mq_queue_reinit(q); + if (q->nr_hw_queues != set->nr_hw_queues) { + pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n", + nr_hw_queues, prev_nr_hw_queues); + set->nr_hw_queues = prev_nr_hw_queues; + blk_mq_map_queues(set); + goto fallback; + } + blk_mq_map_swqueue(q); + } + + list_for_each_entry(q, &set->tag_list, tag_set_list) { + blk_mq_sysfs_register(q); + blk_mq_debugfs_register_hctxs(q); } switch_back: diff --git a/block/blk-pm.c b/block/blk-pm.c new file mode 100644 index 000000000000..f8fdae01bea2 --- /dev/null +++ b/block/blk-pm.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/blk-mq.h> +#include <linux/blk-pm.h> +#include <linux/blkdev.h> +#include <linux/pm_runtime.h> +#include "blk-mq.h" +#include "blk-mq-tag.h" + +/** + * blk_pm_runtime_init - Block layer runtime PM initialization routine + * @q: the queue of the device + * @dev: the device the queue belongs to + * + * Description: + * Initialize runtime-PM-related fields for @q and start auto suspend for + * @dev. Drivers that want to take advantage of request-based runtime PM + * should call this function after @dev has been initialized, and its + * request queue @q has been allocated, and runtime PM for it can not happen + * yet(either due to disabled/forbidden or its usage_count > 0). In most + * cases, driver should call this function before any I/O has taken place. + * + * This function takes care of setting up using auto suspend for the device, + * the autosuspend delay is set to -1 to make runtime suspend impossible + * until an updated value is either set by user or by driver. Drivers do + * not need to touch other autosuspend settings. + * + * The block layer runtime PM is request based, so only works for drivers + * that use request as their IO unit instead of those directly use bio's. + */ +void blk_pm_runtime_init(struct request_queue *q, struct device *dev) +{ + q->dev = dev; + q->rpm_status = RPM_ACTIVE; + pm_runtime_set_autosuspend_delay(q->dev, -1); + pm_runtime_use_autosuspend(q->dev); +} +EXPORT_SYMBOL(blk_pm_runtime_init); + +/** + * blk_pre_runtime_suspend - Pre runtime suspend check + * @q: the queue of the device + * + * Description: + * This function will check if runtime suspend is allowed for the device + * by examining if there are any requests pending in the queue. If there + * are requests pending, the device can not be runtime suspended; otherwise, + * the queue's status will be updated to SUSPENDING and the driver can + * proceed to suspend the device. + * + * For the not allowed case, we mark last busy for the device so that + * runtime PM core will try to autosuspend it some time later. + * + * This function should be called near the start of the device's + * runtime_suspend callback. + * + * Return: + * 0 - OK to runtime suspend the device + * -EBUSY - Device should not be runtime suspended + */ +int blk_pre_runtime_suspend(struct request_queue *q) +{ + int ret = 0; + + if (!q->dev) + return ret; + + WARN_ON_ONCE(q->rpm_status != RPM_ACTIVE); + + /* + * Increase the pm_only counter before checking whether any + * non-PM blk_queue_enter() calls are in progress to avoid that any + * new non-PM blk_queue_enter() calls succeed before the pm_only + * counter is decreased again. + */ + blk_set_pm_only(q); + ret = -EBUSY; + /* Switch q_usage_counter from per-cpu to atomic mode. */ + blk_freeze_queue_start(q); + /* + * Wait until atomic mode has been reached. Since that + * involves calling call_rcu(), it is guaranteed that later + * blk_queue_enter() calls see the pm-only state. See also + * http://lwn.net/Articles/573497/. + */ + percpu_ref_switch_to_atomic_sync(&q->q_usage_counter); + if (percpu_ref_is_zero(&q->q_usage_counter)) + ret = 0; + /* Switch q_usage_counter back to per-cpu mode. */ + blk_mq_unfreeze_queue(q); + + spin_lock_irq(q->queue_lock); + if (ret < 0) + pm_runtime_mark_last_busy(q->dev); + else + q->rpm_status = RPM_SUSPENDING; + spin_unlock_irq(q->queue_lock); + + if (ret) + blk_clear_pm_only(q); + + return ret; +} +EXPORT_SYMBOL(blk_pre_runtime_suspend); + +/** + * blk_post_runtime_suspend - Post runtime suspend processing + * @q: the queue of the device + * @err: return value of the device's runtime_suspend function + * + * Description: + * Update the queue's runtime status according to the return value of the + * device's runtime suspend function and mark last busy for the device so + * that PM core will try to auto suspend the device at a later time. + * + * This function should be called near the end of the device's + * runtime_suspend callback. + */ +void blk_post_runtime_suspend(struct request_queue *q, int err) +{ + if (!q->dev) + return; + + spin_lock_irq(q->queue_lock); + if (!err) { + q->rpm_status = RPM_SUSPENDED; + } else { + q->rpm_status = RPM_ACTIVE; + pm_runtime_mark_last_busy(q->dev); + } + spin_unlock_irq(q->queue_lock); + + if (err) + blk_clear_pm_only(q); +} +EXPORT_SYMBOL(blk_post_runtime_suspend); + +/** + * blk_pre_runtime_resume - Pre runtime resume processing + * @q: the queue of the device + * + * Description: + * Update the queue's runtime status to RESUMING in preparation for the + * runtime resume of the device. + * + * This function should be called near the start of the device's + * runtime_resume callback. + */ +void blk_pre_runtime_resume(struct request_queue *q) +{ + if (!q->dev) + return; + + spin_lock_irq(q->queue_lock); + q->rpm_status = RPM_RESUMING; + spin_unlock_irq(q->queue_lock); +} +EXPORT_SYMBOL(blk_pre_runtime_resume); + +/** + * blk_post_runtime_resume - Post runtime resume processing + * @q: the queue of the device + * @err: return value of the device's runtime_resume function + * + * Description: + * Update the queue's runtime status according to the return value of the + * device's runtime_resume function. If it is successfully resumed, process + * the requests that are queued into the device's queue when it is resuming + * and then mark last busy and initiate autosuspend for it. + * + * This function should be called near the end of the device's + * runtime_resume callback. + */ +void blk_post_runtime_resume(struct request_queue *q, int err) +{ + if (!q->dev) + return; + + spin_lock_irq(q->queue_lock); + if (!err) { + q->rpm_status = RPM_ACTIVE; + pm_runtime_mark_last_busy(q->dev); + pm_request_autosuspend(q->dev); + } else { + q->rpm_status = RPM_SUSPENDED; + } + spin_unlock_irq(q->queue_lock); + + if (!err) + blk_clear_pm_only(q); +} +EXPORT_SYMBOL(blk_post_runtime_resume); + +/** + * blk_set_runtime_active - Force runtime status of the queue to be active + * @q: the queue of the device + * + * If the device is left runtime suspended during system suspend the resume + * hook typically resumes the device and corrects runtime status + * accordingly. However, that does not affect the queue runtime PM status + * which is still "suspended". This prevents processing requests from the + * queue. + * + * This function can be used in driver's resume hook to correct queue + * runtime PM status and re-enable peeking requests from the queue. It + * should be called before first request is added to the queue. + */ +void blk_set_runtime_active(struct request_queue *q) +{ + spin_lock_irq(q->queue_lock); + q->rpm_status = RPM_ACTIVE; + pm_runtime_mark_last_busy(q->dev); + pm_request_autosuspend(q->dev); + spin_unlock_irq(q->queue_lock); +} +EXPORT_SYMBOL(blk_set_runtime_active); diff --git a/block/blk-pm.h b/block/blk-pm.h new file mode 100644 index 000000000000..a8564ea72a41 --- /dev/null +++ b/block/blk-pm.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _BLOCK_BLK_PM_H_ +#define _BLOCK_BLK_PM_H_ + +#include <linux/pm_runtime.h> + +#ifdef CONFIG_PM +static inline void blk_pm_request_resume(struct request_queue *q) +{ + if (q->dev && (q->rpm_status == RPM_SUSPENDED || + q->rpm_status == RPM_SUSPENDING)) + pm_request_resume(q->dev); +} + +static inline void blk_pm_mark_last_busy(struct request *rq) +{ + if (rq->q->dev && !(rq->rq_flags & RQF_PM)) + pm_runtime_mark_last_busy(rq->q->dev); +} + +static inline void blk_pm_requeue_request(struct request *rq) +{ + lockdep_assert_held(rq->q->queue_lock); + + if (rq->q->dev && !(rq->rq_flags & RQF_PM)) + rq->q->nr_pending--; +} + +static inline void blk_pm_add_request(struct request_queue *q, + struct request *rq) +{ + lockdep_assert_held(q->queue_lock); + + if (q->dev && !(rq->rq_flags & RQF_PM)) + q->nr_pending++; +} + +static inline void blk_pm_put_request(struct request *rq) +{ + lockdep_assert_held(rq->q->queue_lock); + + if (rq->q->dev && !(rq->rq_flags & RQF_PM)) + --rq->q->nr_pending; +} +#else +static inline void blk_pm_request_resume(struct request_queue *q) +{ +} + +static inline void blk_pm_mark_last_busy(struct request *rq) +{ +} + +static inline void blk_pm_requeue_request(struct request *rq) +{ +} + +static inline void blk_pm_add_request(struct request_queue *q, + struct request *rq) +{ +} + +static inline void blk_pm_put_request(struct request *rq) +{ +} +#endif + +#endif /* _BLOCK_BLK_PM_H_ */ diff --git a/block/blk-softirq.c b/block/blk-softirq.c index 15c1f5e12eb8..e47a2f751884 100644 --- a/block/blk-softirq.c +++ b/block/blk-softirq.c @@ -97,8 +97,8 @@ static int blk_softirq_cpu_dead(unsigned int cpu) void __blk_complete_request(struct request *req) { - int ccpu, cpu; struct request_queue *q = req->q; + int cpu, ccpu = q->mq_ops ? req->mq_ctx->cpu : req->cpu; unsigned long flags; bool shared = false; @@ -110,8 +110,7 @@ void __blk_complete_request(struct request *req) /* * Select completion CPU */ - if (req->cpu != -1) { - ccpu = req->cpu; + if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags) && ccpu != -1) { if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags)) shared = cpus_share_cache(cpu, ccpu); } else diff --git a/block/blk-stat.c b/block/blk-stat.c index 7587b1c3caaf..90561af85a62 100644 --- a/block/blk-stat.c +++ b/block/blk-stat.c @@ -190,6 +190,7 @@ void blk_stat_enable_accounting(struct request_queue *q) blk_queue_flag_set(QUEUE_FLAG_STATS, q); spin_unlock(&q->stats->lock); } +EXPORT_SYMBOL_GPL(blk_stat_enable_accounting); struct blk_queue_stats *blk_alloc_queue_stats(void) { diff --git a/block/blk-throttle.c b/block/blk-throttle.c index 01d0620a4e4a..4bda70e8db48 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -84,8 +84,7 @@ struct throtl_service_queue { * RB tree of active children throtl_grp's, which are sorted by * their ->disptime. */ - struct rb_root pending_tree; /* RB tree of active tgs */ - struct rb_node *first_pending; /* first node in the tree */ + struct rb_root_cached pending_tree; /* RB tree of active tgs */ unsigned int nr_pending; /* # queued in the tree */ unsigned long first_pending_disptime; /* disptime of the first tg */ struct timer_list pending_timer; /* fires on first_pending_disptime */ @@ -475,7 +474,7 @@ static void throtl_service_queue_init(struct throtl_service_queue *sq) { INIT_LIST_HEAD(&sq->queued[0]); INIT_LIST_HEAD(&sq->queued[1]); - sq->pending_tree = RB_ROOT; + sq->pending_tree = RB_ROOT_CACHED; timer_setup(&sq->pending_timer, throtl_pending_timer_fn, 0); } @@ -616,31 +615,23 @@ static void throtl_pd_free(struct blkg_policy_data *pd) static struct throtl_grp * throtl_rb_first(struct throtl_service_queue *parent_sq) { + struct rb_node *n; /* Service tree is empty */ if (!parent_sq->nr_pending) return NULL; - if (!parent_sq->first_pending) - parent_sq->first_pending = rb_first(&parent_sq->pending_tree); - - if (parent_sq->first_pending) - return rb_entry_tg(parent_sq->first_pending); - - return NULL; -} - -static void rb_erase_init(struct rb_node *n, struct rb_root *root) -{ - rb_erase(n, root); - RB_CLEAR_NODE(n); + n = rb_first_cached(&parent_sq->pending_tree); + WARN_ON_ONCE(!n); + if (!n) + return NULL; + return rb_entry_tg(n); } static void throtl_rb_erase(struct rb_node *n, struct throtl_service_queue *parent_sq) { - if (parent_sq->first_pending == n) - parent_sq->first_pending = NULL; - rb_erase_init(n, &parent_sq->pending_tree); + rb_erase_cached(n, &parent_sq->pending_tree); + RB_CLEAR_NODE(n); --parent_sq->nr_pending; } @@ -658,11 +649,11 @@ static void update_min_dispatch_time(struct throtl_service_queue *parent_sq) static void tg_service_queue_add(struct throtl_grp *tg) { struct throtl_service_queue *parent_sq = tg->service_queue.parent_sq; - struct rb_node **node = &parent_sq->pending_tree.rb_node; + struct rb_node **node = &parent_sq->pending_tree.rb_root.rb_node; struct rb_node *parent = NULL; struct throtl_grp *__tg; unsigned long key = tg->disptime; - int left = 1; + bool leftmost = true; while (*node != NULL) { parent = *node; @@ -672,15 +663,13 @@ static void tg_service_queue_add(struct throtl_grp *tg) node = &parent->rb_left; else { node = &parent->rb_right; - left = 0; + leftmost = false; } } - if (left) - parent_sq->first_pending = &tg->rb_node; - rb_link_node(&tg->rb_node, parent, node); - rb_insert_color(&tg->rb_node, &parent_sq->pending_tree); + rb_insert_color_cached(&tg->rb_node, &parent_sq->pending_tree, + leftmost); } static void __throtl_enqueue_tg(struct throtl_grp *tg) @@ -2126,21 +2115,11 @@ static inline void throtl_update_latency_buckets(struct throtl_data *td) } #endif -static void blk_throtl_assoc_bio(struct throtl_grp *tg, struct bio *bio) -{ -#ifdef CONFIG_BLK_DEV_THROTTLING_LOW - /* fallback to root_blkg if we fail to get a blkg ref */ - if (bio->bi_css && (bio_associate_blkg(bio, tg_to_blkg(tg)) == -ENODEV)) - bio_associate_blkg(bio, bio->bi_disk->queue->root_blkg); - bio_issue_init(&bio->bi_issue, bio_sectors(bio)); -#endif -} - bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg, struct bio *bio) { struct throtl_qnode *qn = NULL; - struct throtl_grp *tg = blkg_to_tg(blkg ?: q->root_blkg); + struct throtl_grp *tg = blkg_to_tg(blkg); struct throtl_service_queue *sq; bool rw = bio_data_dir(bio); bool throttled = false; @@ -2159,7 +2138,6 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg, if (unlikely(blk_queue_bypass(q))) goto out_unlock; - blk_throtl_assoc_bio(tg, bio); blk_throtl_update_idletime(tg); sq = &tg->service_queue; diff --git a/block/blk.h b/block/blk.h index 9db4e389582c..3d2aecba96a4 100644 --- a/block/blk.h +++ b/block/blk.h @@ -4,6 +4,7 @@ #include <linux/idr.h> #include <linux/blk-mq.h> +#include <xen/xen.h> #include "blk-mq.h" /* Amount of time in which a process may batch requests */ @@ -124,7 +125,7 @@ static inline void __blk_get_queue(struct request_queue *q) } struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q, - int node, int cmd_size); + int node, int cmd_size, gfp_t flags); void blk_free_flush_queue(struct blk_flush_queue *q); int blk_init_rl(struct request_list *rl, struct request_queue *q, @@ -149,6 +150,41 @@ static inline void blk_queue_enter_live(struct request_queue *q) percpu_ref_get(&q->q_usage_counter); } +static inline bool biovec_phys_mergeable(struct request_queue *q, + struct bio_vec *vec1, struct bio_vec *vec2) +{ + unsigned long mask = queue_segment_boundary(q); + phys_addr_t addr1 = page_to_phys(vec1->bv_page) + vec1->bv_offset; + phys_addr_t addr2 = page_to_phys(vec2->bv_page) + vec2->bv_offset; + + if (addr1 + vec1->bv_len != addr2) + return false; + if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2)) + return false; + if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask)) + return false; + return true; +} + +static inline bool __bvec_gap_to_prev(struct request_queue *q, + struct bio_vec *bprv, unsigned int offset) +{ + return offset || + ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q)); +} + +/* + * Check if adding a bio_vec after bprv with offset would create a gap in + * the SG list. Most drivers don't care about this, but some do. + */ +static inline bool bvec_gap_to_prev(struct request_queue *q, + struct bio_vec *bprv, unsigned int offset) +{ + if (!queue_virt_boundary(q)) + return false; + return __bvec_gap_to_prev(q, bprv, offset); +} + #ifdef CONFIG_BLK_DEV_INTEGRITY void blk_flush_integrity(void); bool __bio_integrity_endio(struct bio *); @@ -158,7 +194,38 @@ static inline bool bio_integrity_endio(struct bio *bio) return __bio_integrity_endio(bio); return true; } -#else + +static inline bool integrity_req_gap_back_merge(struct request *req, + struct bio *next) +{ + struct bio_integrity_payload *bip = bio_integrity(req->bio); + struct bio_integrity_payload *bip_next = bio_integrity(next); + + return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], + bip_next->bip_vec[0].bv_offset); +} + +static inline bool integrity_req_gap_front_merge(struct request *req, + struct bio *bio) +{ + struct bio_integrity_payload *bip = bio_integrity(bio); + struct bio_integrity_payload *bip_next = bio_integrity(req->bio); + + return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], + bip_next->bip_vec[0].bv_offset); +} +#else /* CONFIG_BLK_DEV_INTEGRITY */ +static inline bool integrity_req_gap_back_merge(struct request *req, + struct bio *next) +{ + return false; +} +static inline bool integrity_req_gap_front_merge(struct request *req, + struct bio *bio) +{ + return false; +} + static inline void blk_flush_integrity(void) { } @@ -166,7 +233,7 @@ static inline bool bio_integrity_endio(struct bio *bio) { return true; } -#endif +#endif /* CONFIG_BLK_DEV_INTEGRITY */ void blk_timeout_work(struct work_struct *work); unsigned long blk_rq_timeout(unsigned long timeout); diff --git a/block/bounce.c b/block/bounce.c index bc63b3a2d18c..ec0d99995f5f 100644 --- a/block/bounce.c +++ b/block/bounce.c @@ -31,6 +31,24 @@ static struct bio_set bounce_bio_set, bounce_bio_split; static mempool_t page_pool, isa_page_pool; +static void init_bounce_bioset(void) +{ + static bool bounce_bs_setup; + int ret; + + if (bounce_bs_setup) + return; + + ret = bioset_init(&bounce_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS); + BUG_ON(ret); + if (bioset_integrity_create(&bounce_bio_set, BIO_POOL_SIZE)) + BUG_ON(1); + + ret = bioset_init(&bounce_bio_split, BIO_POOL_SIZE, 0, 0); + BUG_ON(ret); + bounce_bs_setup = true; +} + #if defined(CONFIG_HIGHMEM) static __init int init_emergency_pool(void) { @@ -44,14 +62,7 @@ static __init int init_emergency_pool(void) BUG_ON(ret); pr_info("pool size: %d pages\n", POOL_SIZE); - ret = bioset_init(&bounce_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS); - BUG_ON(ret); - if (bioset_integrity_create(&bounce_bio_set, BIO_POOL_SIZE)) - BUG_ON(1); - - ret = bioset_init(&bounce_bio_split, BIO_POOL_SIZE, 0, 0); - BUG_ON(ret); - + init_bounce_bioset(); return 0; } @@ -86,6 +97,8 @@ static void *mempool_alloc_pages_isa(gfp_t gfp_mask, void *data) return mempool_alloc_pages(gfp_mask | GFP_DMA, data); } +static DEFINE_MUTEX(isa_mutex); + /* * gets called "every" time someone init's a queue with BLK_BOUNCE_ISA * as the max address, so check if the pool has already been created. @@ -94,14 +107,20 @@ int init_emergency_isa_pool(void) { int ret; - if (mempool_initialized(&isa_page_pool)) + mutex_lock(&isa_mutex); + + if (mempool_initialized(&isa_page_pool)) { + mutex_unlock(&isa_mutex); return 0; + } ret = mempool_init(&isa_page_pool, ISA_POOL_SIZE, mempool_alloc_pages_isa, mempool_free_pages, (void *) 0); BUG_ON(ret); pr_info("isa pool size: %d pages\n", ISA_POOL_SIZE); + init_bounce_bioset(); + mutex_unlock(&isa_mutex); return 0; } @@ -257,7 +276,9 @@ static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask, } } - bio_clone_blkcg_association(bio, bio_src); + bio_clone_blkg_association(bio, bio_src); + + blkcg_bio_issue_init(bio); return bio; } diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 2eb87444b157..6a3d87dd3c1a 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -1644,14 +1644,20 @@ static void cfq_pd_offline(struct blkg_policy_data *pd) int i; for (i = 0; i < IOPRIO_BE_NR; i++) { - if (cfqg->async_cfqq[0][i]) + if (cfqg->async_cfqq[0][i]) { cfq_put_queue(cfqg->async_cfqq[0][i]); - if (cfqg->async_cfqq[1][i]) + cfqg->async_cfqq[0][i] = NULL; + } + if (cfqg->async_cfqq[1][i]) { cfq_put_queue(cfqg->async_cfqq[1][i]); + cfqg->async_cfqq[1][i] = NULL; + } } - if (cfqg->async_idle_cfqq) + if (cfqg->async_idle_cfqq) { cfq_put_queue(cfqg->async_idle_cfqq); + cfqg->async_idle_cfqq = NULL; + } /* * @blkg is going offline and will be ignored by @@ -3753,7 +3759,7 @@ static void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio) uint64_t serial_nr; rcu_read_lock(); - serial_nr = bio_blkcg(bio)->css.serial_nr; + serial_nr = __bio_blkcg(bio)->css.serial_nr; rcu_read_unlock(); /* @@ -3818,7 +3824,7 @@ cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic, struct cfq_group *cfqg; rcu_read_lock(); - cfqg = cfq_lookup_cfqg(cfqd, bio_blkcg(bio)); + cfqg = cfq_lookup_cfqg(cfqd, __bio_blkcg(bio)); if (!cfqg) { cfqq = &cfqd->oom_cfqq; goto out; diff --git a/block/elevator.c b/block/elevator.c index fae58b2f906f..8fdcd64ae12e 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -41,6 +41,7 @@ #include "blk.h" #include "blk-mq-sched.h" +#include "blk-pm.h" #include "blk-wbt.h" static DEFINE_SPINLOCK(elv_list_lock); @@ -557,27 +558,6 @@ void elv_bio_merged(struct request_queue *q, struct request *rq, e->type->ops.sq.elevator_bio_merged_fn(q, rq, bio); } -#ifdef CONFIG_PM -static void blk_pm_requeue_request(struct request *rq) -{ - if (rq->q->dev && !(rq->rq_flags & RQF_PM)) - rq->q->nr_pending--; -} - -static void blk_pm_add_request(struct request_queue *q, struct request *rq) -{ - if (q->dev && !(rq->rq_flags & RQF_PM) && q->nr_pending++ == 0 && - (q->rpm_status == RPM_SUSPENDED || q->rpm_status == RPM_SUSPENDING)) - pm_request_resume(q->dev); -} -#else -static inline void blk_pm_requeue_request(struct request *rq) {} -static inline void blk_pm_add_request(struct request_queue *q, - struct request *rq) -{ -} -#endif - void elv_requeue_request(struct request_queue *q, struct request *rq) { /* diff --git a/block/genhd.c b/block/genhd.c index be5bab20b2ab..cff6bdf27226 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -567,7 +567,8 @@ static int exact_lock(dev_t devt, void *data) return 0; } -static void register_disk(struct device *parent, struct gendisk *disk) +static void register_disk(struct device *parent, struct gendisk *disk, + const struct attribute_group **groups) { struct device *ddev = disk_to_dev(disk); struct block_device *bdev; @@ -582,6 +583,10 @@ static void register_disk(struct device *parent, struct gendisk *disk) /* delay uevents, until we scanned partition table */ dev_set_uevent_suppress(ddev, 1); + if (groups) { + WARN_ON(ddev->groups); + ddev->groups = groups; + } if (device_add(ddev)) return; if (!sysfs_deprecated) { @@ -647,6 +652,7 @@ exit: * __device_add_disk - add disk information to kernel list * @parent: parent device for the disk * @disk: per-device partitioning information + * @groups: Additional per-device sysfs groups * @register_queue: register the queue if set to true * * This function registers the partitioning information in @disk @@ -655,6 +661,7 @@ exit: * FIXME: error handling */ static void __device_add_disk(struct device *parent, struct gendisk *disk, + const struct attribute_group **groups, bool register_queue) { dev_t devt; @@ -698,7 +705,7 @@ static void __device_add_disk(struct device *parent, struct gendisk *disk, blk_register_region(disk_devt(disk), disk->minors, NULL, exact_match, exact_lock, disk); } - register_disk(parent, disk); + register_disk(parent, disk, groups); if (register_queue) blk_register_queue(disk); @@ -712,15 +719,17 @@ static void __device_add_disk(struct device *parent, struct gendisk *disk, blk_integrity_add(disk); } -void device_add_disk(struct device *parent, struct gendisk *disk) +void device_add_disk(struct device *parent, struct gendisk *disk, + const struct attribute_group **groups) + { - __device_add_disk(parent, disk, true); + __device_add_disk(parent, disk, groups, true); } EXPORT_SYMBOL(device_add_disk); void device_add_disk_no_queue_reg(struct device *parent, struct gendisk *disk) { - __device_add_disk(parent, disk, false); + __device_add_disk(parent, disk, NULL, false); } EXPORT_SYMBOL(device_add_disk_no_queue_reg); diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index a1660bafc912..eccac01a10b6 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -29,19 +29,30 @@ #include "blk-mq-debugfs.h" #include "blk-mq-sched.h" #include "blk-mq-tag.h" -#include "blk-stat.h" -/* Scheduling domains. */ +#define CREATE_TRACE_POINTS +#include <trace/events/kyber.h> + +/* + * Scheduling domains: the device is divided into multiple domains based on the + * request type. + */ enum { KYBER_READ, - KYBER_SYNC_WRITE, - KYBER_OTHER, /* Async writes, discard, etc. */ + KYBER_WRITE, + KYBER_DISCARD, + KYBER_OTHER, KYBER_NUM_DOMAINS, }; -enum { - KYBER_MIN_DEPTH = 256, +static const char *kyber_domain_names[] = { + [KYBER_READ] = "READ", + [KYBER_WRITE] = "WRITE", + [KYBER_DISCARD] = "DISCARD", + [KYBER_OTHER] = "OTHER", +}; +enum { /* * In order to prevent starvation of synchronous requests by a flood of * asynchronous requests, we reserve 25% of requests for synchronous @@ -51,25 +62,87 @@ enum { }; /* - * Initial device-wide depths for each scheduling domain. + * Maximum device-wide depth for each scheduling domain. * - * Even for fast devices with lots of tags like NVMe, you can saturate - * the device with only a fraction of the maximum possible queue depth. - * So, we cap these to a reasonable value. + * Even for fast devices with lots of tags like NVMe, you can saturate the + * device with only a fraction of the maximum possible queue depth. So, we cap + * these to a reasonable value. */ static const unsigned int kyber_depth[] = { [KYBER_READ] = 256, - [KYBER_SYNC_WRITE] = 128, - [KYBER_OTHER] = 64, + [KYBER_WRITE] = 128, + [KYBER_DISCARD] = 64, + [KYBER_OTHER] = 16, }; /* - * Scheduling domain batch sizes. We favor reads. + * Default latency targets for each scheduling domain. + */ +static const u64 kyber_latency_targets[] = { + [KYBER_READ] = 2ULL * NSEC_PER_MSEC, + [KYBER_WRITE] = 10ULL * NSEC_PER_MSEC, + [KYBER_DISCARD] = 5ULL * NSEC_PER_SEC, +}; + +/* + * Batch size (number of requests we'll dispatch in a row) for each scheduling + * domain. */ static const unsigned int kyber_batch_size[] = { [KYBER_READ] = 16, - [KYBER_SYNC_WRITE] = 8, - [KYBER_OTHER] = 8, + [KYBER_WRITE] = 8, + [KYBER_DISCARD] = 1, + [KYBER_OTHER] = 1, +}; + +/* + * Requests latencies are recorded in a histogram with buckets defined relative + * to the target latency: + * + * <= 1/4 * target latency + * <= 1/2 * target latency + * <= 3/4 * target latency + * <= target latency + * <= 1 1/4 * target latency + * <= 1 1/2 * target latency + * <= 1 3/4 * target latency + * > 1 3/4 * target latency + */ +enum { + /* + * The width of the latency histogram buckets is + * 1 / (1 << KYBER_LATENCY_SHIFT) * target latency. + */ + KYBER_LATENCY_SHIFT = 2, + /* + * The first (1 << KYBER_LATENCY_SHIFT) buckets are <= target latency, + * thus, "good". + */ + KYBER_GOOD_BUCKETS = 1 << KYBER_LATENCY_SHIFT, + /* There are also (1 << KYBER_LATENCY_SHIFT) "bad" buckets. */ + KYBER_LATENCY_BUCKETS = 2 << KYBER_LATENCY_SHIFT, +}; + +/* + * We measure both the total latency and the I/O latency (i.e., latency after + * submitting to the device). + */ +enum { + KYBER_TOTAL_LATENCY, + KYBER_IO_LATENCY, +}; + +static const char *kyber_latency_type_names[] = { + [KYBER_TOTAL_LATENCY] = "total", + [KYBER_IO_LATENCY] = "I/O", +}; + +/* + * Per-cpu latency histograms: total latency and I/O latency for each scheduling + * domain except for KYBER_OTHER. + */ +struct kyber_cpu_latency { + atomic_t buckets[KYBER_OTHER][2][KYBER_LATENCY_BUCKETS]; }; /* @@ -88,12 +161,9 @@ struct kyber_ctx_queue { struct kyber_queue_data { struct request_queue *q; - struct blk_stat_callback *cb; - /* - * The device is divided into multiple scheduling domains based on the - * request type. Each domain has a fixed number of in-flight requests of - * that type device-wide, limited by these tokens. + * Each scheduling domain has a limited number of in-flight requests + * device-wide, limited by these tokens. */ struct sbitmap_queue domain_tokens[KYBER_NUM_DOMAINS]; @@ -103,8 +173,19 @@ struct kyber_queue_data { */ unsigned int async_depth; + struct kyber_cpu_latency __percpu *cpu_latency; + + /* Timer for stats aggregation and adjusting domain tokens. */ + struct timer_list timer; + + unsigned int latency_buckets[KYBER_OTHER][2][KYBER_LATENCY_BUCKETS]; + + unsigned long latency_timeout[KYBER_OTHER]; + + int domain_p99[KYBER_OTHER]; + /* Target latencies in nanoseconds. */ - u64 read_lat_nsec, write_lat_nsec; + u64 latency_targets[KYBER_OTHER]; }; struct kyber_hctx_data { @@ -124,233 +205,219 @@ static int kyber_domain_wake(wait_queue_entry_t *wait, unsigned mode, int flags, static unsigned int kyber_sched_domain(unsigned int op) { - if ((op & REQ_OP_MASK) == REQ_OP_READ) + switch (op & REQ_OP_MASK) { + case REQ_OP_READ: return KYBER_READ; - else if ((op & REQ_OP_MASK) == REQ_OP_WRITE && op_is_sync(op)) - return KYBER_SYNC_WRITE; - else + case REQ_OP_WRITE: + return KYBER_WRITE; + case REQ_OP_DISCARD: + return KYBER_DISCARD; + default: return KYBER_OTHER; + } } -enum { - NONE = 0, - GOOD = 1, - GREAT = 2, - BAD = -1, - AWFUL = -2, -}; - -#define IS_GOOD(status) ((status) > 0) -#define IS_BAD(status) ((status) < 0) - -static int kyber_lat_status(struct blk_stat_callback *cb, - unsigned int sched_domain, u64 target) +static void flush_latency_buckets(struct kyber_queue_data *kqd, + struct kyber_cpu_latency *cpu_latency, + unsigned int sched_domain, unsigned int type) { - u64 latency; - - if (!cb->stat[sched_domain].nr_samples) - return NONE; + unsigned int *buckets = kqd->latency_buckets[sched_domain][type]; + atomic_t *cpu_buckets = cpu_latency->buckets[sched_domain][type]; + unsigned int bucket; - latency = cb->stat[sched_domain].mean; - if (latency >= 2 * target) - return AWFUL; - else if (latency > target) - return BAD; - else if (latency <= target / 2) - return GREAT; - else /* (latency <= target) */ - return GOOD; + for (bucket = 0; bucket < KYBER_LATENCY_BUCKETS; bucket++) + buckets[bucket] += atomic_xchg(&cpu_buckets[bucket], 0); } /* - * Adjust the read or synchronous write depth given the status of reads and - * writes. The goal is that the latencies of the two domains are fair (i.e., if - * one is good, then the other is good). + * Calculate the histogram bucket with the given percentile rank, or -1 if there + * aren't enough samples yet. */ -static void kyber_adjust_rw_depth(struct kyber_queue_data *kqd, - unsigned int sched_domain, int this_status, - int other_status) +static int calculate_percentile(struct kyber_queue_data *kqd, + unsigned int sched_domain, unsigned int type, + unsigned int percentile) { - unsigned int orig_depth, depth; + unsigned int *buckets = kqd->latency_buckets[sched_domain][type]; + unsigned int bucket, samples = 0, percentile_samples; + + for (bucket = 0; bucket < KYBER_LATENCY_BUCKETS; bucket++) + samples += buckets[bucket]; + + if (!samples) + return -1; /* - * If this domain had no samples, or reads and writes are both good or - * both bad, don't adjust the depth. + * We do the calculation once we have 500 samples or one second passes + * since the first sample was recorded, whichever comes first. */ - if (this_status == NONE || - (IS_GOOD(this_status) && IS_GOOD(other_status)) || - (IS_BAD(this_status) && IS_BAD(other_status))) - return; - - orig_depth = depth = kqd->domain_tokens[sched_domain].sb.depth; + if (!kqd->latency_timeout[sched_domain]) + kqd->latency_timeout[sched_domain] = max(jiffies + HZ, 1UL); + if (samples < 500 && + time_is_after_jiffies(kqd->latency_timeout[sched_domain])) { + return -1; + } + kqd->latency_timeout[sched_domain] = 0; - if (other_status == NONE) { - depth++; - } else { - switch (this_status) { - case GOOD: - if (other_status == AWFUL) - depth -= max(depth / 4, 1U); - else - depth -= max(depth / 8, 1U); - break; - case GREAT: - if (other_status == AWFUL) - depth /= 2; - else - depth -= max(depth / 4, 1U); + percentile_samples = DIV_ROUND_UP(samples * percentile, 100); + for (bucket = 0; bucket < KYBER_LATENCY_BUCKETS - 1; bucket++) { + if (buckets[bucket] >= percentile_samples) break; - case BAD: - depth++; - break; - case AWFUL: - if (other_status == GREAT) - depth += 2; - else - depth++; - break; - } + percentile_samples -= buckets[bucket]; } + memset(buckets, 0, sizeof(kqd->latency_buckets[sched_domain][type])); - depth = clamp(depth, 1U, kyber_depth[sched_domain]); - if (depth != orig_depth) - sbitmap_queue_resize(&kqd->domain_tokens[sched_domain], depth); + trace_kyber_latency(kqd->q, kyber_domain_names[sched_domain], + kyber_latency_type_names[type], percentile, + bucket + 1, 1 << KYBER_LATENCY_SHIFT, samples); + + return bucket; } -/* - * Adjust the depth of other requests given the status of reads and synchronous - * writes. As long as either domain is doing fine, we don't throttle, but if - * both domains are doing badly, we throttle heavily. - */ -static void kyber_adjust_other_depth(struct kyber_queue_data *kqd, - int read_status, int write_status, - bool have_samples) -{ - unsigned int orig_depth, depth; - int status; - - orig_depth = depth = kqd->domain_tokens[KYBER_OTHER].sb.depth; - - if (read_status == NONE && write_status == NONE) { - depth += 2; - } else if (have_samples) { - if (read_status == NONE) - status = write_status; - else if (write_status == NONE) - status = read_status; - else - status = max(read_status, write_status); - switch (status) { - case GREAT: - depth += 2; - break; - case GOOD: - depth++; - break; - case BAD: - depth -= max(depth / 4, 1U); - break; - case AWFUL: - depth /= 2; - break; - } +static void kyber_resize_domain(struct kyber_queue_data *kqd, + unsigned int sched_domain, unsigned int depth) +{ + depth = clamp(depth, 1U, kyber_depth[sched_domain]); + if (depth != kqd->domain_tokens[sched_domain].sb.depth) { + sbitmap_queue_resize(&kqd->domain_tokens[sched_domain], depth); + trace_kyber_adjust(kqd->q, kyber_domain_names[sched_domain], + depth); } - - depth = clamp(depth, 1U, kyber_depth[KYBER_OTHER]); - if (depth != orig_depth) - sbitmap_queue_resize(&kqd->domain_tokens[KYBER_OTHER], depth); } -/* - * Apply heuristics for limiting queue depths based on gathered latency - * statistics. - */ -static void kyber_stat_timer_fn(struct blk_stat_callback *cb) +static void kyber_timer_fn(struct timer_list *t) { - struct kyber_queue_data *kqd = cb->data; - int read_status, write_status; + struct kyber_queue_data *kqd = from_timer(kqd, t, timer); + unsigned int sched_domain; + int cpu; + bool bad = false; + + /* Sum all of the per-cpu latency histograms. */ + for_each_online_cpu(cpu) { + struct kyber_cpu_latency *cpu_latency; + + cpu_latency = per_cpu_ptr(kqd->cpu_latency, cpu); + for (sched_domain = 0; sched_domain < KYBER_OTHER; sched_domain++) { + flush_latency_buckets(kqd, cpu_latency, sched_domain, + KYBER_TOTAL_LATENCY); + flush_latency_buckets(kqd, cpu_latency, sched_domain, + KYBER_IO_LATENCY); + } + } - read_status = kyber_lat_status(cb, KYBER_READ, kqd->read_lat_nsec); - write_status = kyber_lat_status(cb, KYBER_SYNC_WRITE, kqd->write_lat_nsec); + /* + * Check if any domains have a high I/O latency, which might indicate + * congestion in the device. Note that we use the p90; we don't want to + * be too sensitive to outliers here. + */ + for (sched_domain = 0; sched_domain < KYBER_OTHER; sched_domain++) { + int p90; - kyber_adjust_rw_depth(kqd, KYBER_READ, read_status, write_status); - kyber_adjust_rw_depth(kqd, KYBER_SYNC_WRITE, write_status, read_status); - kyber_adjust_other_depth(kqd, read_status, write_status, - cb->stat[KYBER_OTHER].nr_samples != 0); + p90 = calculate_percentile(kqd, sched_domain, KYBER_IO_LATENCY, + 90); + if (p90 >= KYBER_GOOD_BUCKETS) + bad = true; + } /* - * Continue monitoring latencies if we aren't hitting the targets or - * we're still throttling other requests. + * Adjust the scheduling domain depths. If we determined that there was + * congestion, we throttle all domains with good latencies. Either way, + * we ease up on throttling domains with bad latencies. */ - if (!blk_stat_is_active(kqd->cb) && - ((IS_BAD(read_status) || IS_BAD(write_status) || - kqd->domain_tokens[KYBER_OTHER].sb.depth < kyber_depth[KYBER_OTHER]))) - blk_stat_activate_msecs(kqd->cb, 100); + for (sched_domain = 0; sched_domain < KYBER_OTHER; sched_domain++) { + unsigned int orig_depth, depth; + int p99; + + p99 = calculate_percentile(kqd, sched_domain, + KYBER_TOTAL_LATENCY, 99); + /* + * This is kind of subtle: different domains will not + * necessarily have enough samples to calculate the latency + * percentiles during the same window, so we have to remember + * the p99 for the next time we observe congestion; once we do, + * we don't want to throttle again until we get more data, so we + * reset it to -1. + */ + if (bad) { + if (p99 < 0) + p99 = kqd->domain_p99[sched_domain]; + kqd->domain_p99[sched_domain] = -1; + } else if (p99 >= 0) { + kqd->domain_p99[sched_domain] = p99; + } + if (p99 < 0) + continue; + + /* + * If this domain has bad latency, throttle less. Otherwise, + * throttle more iff we determined that there is congestion. + * + * The new depth is scaled linearly with the p99 latency vs the + * latency target. E.g., if the p99 is 3/4 of the target, then + * we throttle down to 3/4 of the current depth, and if the p99 + * is 2x the target, then we double the depth. + */ + if (bad || p99 >= KYBER_GOOD_BUCKETS) { + orig_depth = kqd->domain_tokens[sched_domain].sb.depth; + depth = (orig_depth * (p99 + 1)) >> KYBER_LATENCY_SHIFT; + kyber_resize_domain(kqd, sched_domain, depth); + } + } } -static unsigned int kyber_sched_tags_shift(struct kyber_queue_data *kqd) +static unsigned int kyber_sched_tags_shift(struct request_queue *q) { /* * All of the hardware queues have the same depth, so we can just grab * the shift of the first one. */ - return kqd->q->queue_hw_ctx[0]->sched_tags->bitmap_tags.sb.shift; -} - -static int kyber_bucket_fn(const struct request *rq) -{ - return kyber_sched_domain(rq->cmd_flags); + return q->queue_hw_ctx[0]->sched_tags->bitmap_tags.sb.shift; } static struct kyber_queue_data *kyber_queue_data_alloc(struct request_queue *q) { struct kyber_queue_data *kqd; - unsigned int max_tokens; unsigned int shift; int ret = -ENOMEM; int i; - kqd = kmalloc_node(sizeof(*kqd), GFP_KERNEL, q->node); + kqd = kzalloc_node(sizeof(*kqd), GFP_KERNEL, q->node); if (!kqd) goto err; + kqd->q = q; - kqd->cb = blk_stat_alloc_callback(kyber_stat_timer_fn, kyber_bucket_fn, - KYBER_NUM_DOMAINS, kqd); - if (!kqd->cb) + kqd->cpu_latency = alloc_percpu_gfp(struct kyber_cpu_latency, + GFP_KERNEL | __GFP_ZERO); + if (!kqd->cpu_latency) goto err_kqd; - /* - * The maximum number of tokens for any scheduling domain is at least - * the queue depth of a single hardware queue. If the hardware doesn't - * have many tags, still provide a reasonable number. - */ - max_tokens = max_t(unsigned int, q->tag_set->queue_depth, - KYBER_MIN_DEPTH); + timer_setup(&kqd->timer, kyber_timer_fn, 0); + for (i = 0; i < KYBER_NUM_DOMAINS; i++) { WARN_ON(!kyber_depth[i]); WARN_ON(!kyber_batch_size[i]); ret = sbitmap_queue_init_node(&kqd->domain_tokens[i], - max_tokens, -1, false, GFP_KERNEL, - q->node); + kyber_depth[i], -1, false, + GFP_KERNEL, q->node); if (ret) { while (--i >= 0) sbitmap_queue_free(&kqd->domain_tokens[i]); - goto err_cb; + goto err_buckets; } - sbitmap_queue_resize(&kqd->domain_tokens[i], kyber_depth[i]); } - shift = kyber_sched_tags_shift(kqd); - kqd->async_depth = (1U << shift) * KYBER_ASYNC_PERCENT / 100U; + for (i = 0; i < KYBER_OTHER; i++) { + kqd->domain_p99[i] = -1; + kqd->latency_targets[i] = kyber_latency_targets[i]; + } - kqd->read_lat_nsec = 2000000ULL; - kqd->write_lat_nsec = 10000000ULL; + shift = kyber_sched_tags_shift(q); + kqd->async_depth = (1U << shift) * KYBER_ASYNC_PERCENT / 100U; return kqd; -err_cb: - blk_stat_free_callback(kqd->cb); +err_buckets: + free_percpu(kqd->cpu_latency); err_kqd: kfree(kqd); err: @@ -372,25 +439,24 @@ static int kyber_init_sched(struct request_queue *q, struct elevator_type *e) return PTR_ERR(kqd); } + blk_stat_enable_accounting(q); + eq->elevator_data = kqd; q->elevator = eq; - blk_stat_add_callback(q, kqd->cb); - return 0; } static void kyber_exit_sched(struct elevator_queue *e) { struct kyber_queue_data *kqd = e->elevator_data; - struct request_queue *q = kqd->q; int i; - blk_stat_remove_callback(q, kqd->cb); + del_timer_sync(&kqd->timer); for (i = 0; i < KYBER_NUM_DOMAINS; i++) sbitmap_queue_free(&kqd->domain_tokens[i]); - blk_stat_free_callback(kqd->cb); + free_percpu(kqd->cpu_latency); kfree(kqd); } @@ -558,41 +624,44 @@ static void kyber_finish_request(struct request *rq) rq_clear_domain_token(kqd, rq); } -static void kyber_completed_request(struct request *rq) +static void add_latency_sample(struct kyber_cpu_latency *cpu_latency, + unsigned int sched_domain, unsigned int type, + u64 target, u64 latency) { - struct request_queue *q = rq->q; - struct kyber_queue_data *kqd = q->elevator->elevator_data; - unsigned int sched_domain; - u64 now, latency, target; + unsigned int bucket; + u64 divisor; - /* - * Check if this request met our latency goal. If not, quickly gather - * some statistics and start throttling. - */ - sched_domain = kyber_sched_domain(rq->cmd_flags); - switch (sched_domain) { - case KYBER_READ: - target = kqd->read_lat_nsec; - break; - case KYBER_SYNC_WRITE: - target = kqd->write_lat_nsec; - break; - default: - return; + if (latency > 0) { + divisor = max_t(u64, target >> KYBER_LATENCY_SHIFT, 1); + bucket = min_t(unsigned int, div64_u64(latency - 1, divisor), + KYBER_LATENCY_BUCKETS - 1); + } else { + bucket = 0; } - /* If we are already monitoring latencies, don't check again. */ - if (blk_stat_is_active(kqd->cb)) - return; + atomic_inc(&cpu_latency->buckets[sched_domain][type][bucket]); +} - now = ktime_get_ns(); - if (now < rq->io_start_time_ns) +static void kyber_completed_request(struct request *rq, u64 now) +{ + struct kyber_queue_data *kqd = rq->q->elevator->elevator_data; + struct kyber_cpu_latency *cpu_latency; + unsigned int sched_domain; + u64 target; + + sched_domain = kyber_sched_domain(rq->cmd_flags); + if (sched_domain == KYBER_OTHER) return; - latency = now - rq->io_start_time_ns; + cpu_latency = get_cpu_ptr(kqd->cpu_latency); + target = kqd->latency_targets[sched_domain]; + add_latency_sample(cpu_latency, sched_domain, KYBER_TOTAL_LATENCY, + target, now - rq->start_time_ns); + add_latency_sample(cpu_latency, sched_domain, KYBER_IO_LATENCY, target, + now - rq->io_start_time_ns); + put_cpu_ptr(kqd->cpu_latency); - if (latency > target) - blk_stat_activate_msecs(kqd->cb, 10); + timer_reduce(&kqd->timer, jiffies + HZ / 10); } struct flush_kcq_data { @@ -713,6 +782,9 @@ kyber_dispatch_cur_domain(struct kyber_queue_data *kqd, rq_set_domain_token(rq, nr); list_del_init(&rq->queuelist); return rq; + } else { + trace_kyber_throttled(kqd->q, + kyber_domain_names[khd->cur_domain]); } } else if (sbitmap_any_bit_set(&khd->kcq_map[khd->cur_domain])) { nr = kyber_get_domain_token(kqd, khd, hctx); @@ -723,6 +795,9 @@ kyber_dispatch_cur_domain(struct kyber_queue_data *kqd, rq_set_domain_token(rq, nr); list_del_init(&rq->queuelist); return rq; + } else { + trace_kyber_throttled(kqd->q, + kyber_domain_names[khd->cur_domain]); } } @@ -790,17 +865,17 @@ static bool kyber_has_work(struct blk_mq_hw_ctx *hctx) return false; } -#define KYBER_LAT_SHOW_STORE(op) \ -static ssize_t kyber_##op##_lat_show(struct elevator_queue *e, \ - char *page) \ +#define KYBER_LAT_SHOW_STORE(domain, name) \ +static ssize_t kyber_##name##_lat_show(struct elevator_queue *e, \ + char *page) \ { \ struct kyber_queue_data *kqd = e->elevator_data; \ \ - return sprintf(page, "%llu\n", kqd->op##_lat_nsec); \ + return sprintf(page, "%llu\n", kqd->latency_targets[domain]); \ } \ \ -static ssize_t kyber_##op##_lat_store(struct elevator_queue *e, \ - const char *page, size_t count) \ +static ssize_t kyber_##name##_lat_store(struct elevator_queue *e, \ + const char *page, size_t count) \ { \ struct kyber_queue_data *kqd = e->elevator_data; \ unsigned long long nsec; \ @@ -810,12 +885,12 @@ static ssize_t kyber_##op##_lat_store(struct elevator_queue *e, \ if (ret) \ return ret; \ \ - kqd->op##_lat_nsec = nsec; \ + kqd->latency_targets[domain] = nsec; \ \ return count; \ } -KYBER_LAT_SHOW_STORE(read); -KYBER_LAT_SHOW_STORE(write); +KYBER_LAT_SHOW_STORE(KYBER_READ, read); +KYBER_LAT_SHOW_STORE(KYBER_WRITE, write); #undef KYBER_LAT_SHOW_STORE #define KYBER_LAT_ATTR(op) __ATTR(op##_lat_nsec, 0644, kyber_##op##_lat_show, kyber_##op##_lat_store) @@ -882,7 +957,8 @@ static int kyber_##name##_waiting_show(void *data, struct seq_file *m) \ return 0; \ } KYBER_DEBUGFS_DOMAIN_ATTRS(KYBER_READ, read) -KYBER_DEBUGFS_DOMAIN_ATTRS(KYBER_SYNC_WRITE, sync_write) +KYBER_DEBUGFS_DOMAIN_ATTRS(KYBER_WRITE, write) +KYBER_DEBUGFS_DOMAIN_ATTRS(KYBER_DISCARD, discard) KYBER_DEBUGFS_DOMAIN_ATTRS(KYBER_OTHER, other) #undef KYBER_DEBUGFS_DOMAIN_ATTRS @@ -900,20 +976,7 @@ static int kyber_cur_domain_show(void *data, struct seq_file *m) struct blk_mq_hw_ctx *hctx = data; struct kyber_hctx_data *khd = hctx->sched_data; - switch (khd->cur_domain) { - case KYBER_READ: - seq_puts(m, "READ\n"); - break; - case KYBER_SYNC_WRITE: - seq_puts(m, "SYNC_WRITE\n"); - break; - case KYBER_OTHER: - seq_puts(m, "OTHER\n"); - break; - default: - seq_printf(m, "%u\n", khd->cur_domain); - break; - } + seq_printf(m, "%s\n", kyber_domain_names[khd->cur_domain]); return 0; } @@ -930,7 +993,8 @@ static int kyber_batching_show(void *data, struct seq_file *m) {#name "_tokens", 0400, kyber_##name##_tokens_show} static const struct blk_mq_debugfs_attr kyber_queue_debugfs_attrs[] = { KYBER_QUEUE_DOMAIN_ATTRS(read), - KYBER_QUEUE_DOMAIN_ATTRS(sync_write), + KYBER_QUEUE_DOMAIN_ATTRS(write), + KYBER_QUEUE_DOMAIN_ATTRS(discard), KYBER_QUEUE_DOMAIN_ATTRS(other), {"async_depth", 0400, kyber_async_depth_show}, {}, @@ -942,7 +1006,8 @@ static const struct blk_mq_debugfs_attr kyber_queue_debugfs_attrs[] = { {#name "_waiting", 0400, kyber_##name##_waiting_show} static const struct blk_mq_debugfs_attr kyber_hctx_debugfs_attrs[] = { KYBER_HCTX_DOMAIN_ATTRS(read), - KYBER_HCTX_DOMAIN_ATTRS(sync_write), + KYBER_HCTX_DOMAIN_ATTRS(write), + KYBER_HCTX_DOMAIN_ATTRS(discard), KYBER_HCTX_DOMAIN_ATTRS(other), {"cur_domain", 0400, kyber_cur_domain_show}, {"batching", 0400, kyber_batching_show}, diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index 08f26db2da7e..2a361e22d38d 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -1428,7 +1428,7 @@ static int __init iort_add_platform_device(struct acpi_iort_node *node, return 0; dma_deconfigure: - acpi_dma_deconfigure(&pdev->dev); + arch_teardown_dma_ops(&pdev->dev); dev_put: platform_device_put(pdev); diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index e1b6231cfa1c..56676a56b3e3 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -1469,16 +1469,6 @@ int acpi_dma_configure(struct device *dev, enum dev_dma_attr attr) } EXPORT_SYMBOL_GPL(acpi_dma_configure); -/** - * acpi_dma_deconfigure - Tear-down DMA configuration for the device. - * @dev: The pointer to the device - */ -void acpi_dma_deconfigure(struct device *dev) -{ - arch_teardown_dma_ops(dev); -} -EXPORT_SYMBOL_GPL(acpi_dma_deconfigure); - static void acpi_init_coherency(struct acpi_device *adev) { unsigned long long cca = 0; diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig index 39b181d6bd0d..4ca7a6b4eaae 100644 --- a/drivers/ata/Kconfig +++ b/drivers/ata/Kconfig @@ -33,7 +33,6 @@ if ATA config ATA_NONSTANDARD bool - default n config ATA_VERBOSE_ERROR bool "Verbose ATA error reporting" @@ -62,7 +61,6 @@ config ATA_ACPI config SATA_ZPODD bool "SATA Zero Power Optical Disc Drive (ZPODD) support" depends on ATA_ACPI && PM - default n help This option adds support for SATA Zero Power Optical Disc Drive (ZPODD). It requires both the ODD and the platform @@ -121,7 +119,8 @@ config SATA_AHCI_PLATFORM config AHCI_BRCM tristate "Broadcom AHCI SATA support" - depends on ARCH_BRCMSTB || BMIPS_GENERIC || ARCH_BCM_NSP + depends on ARCH_BRCMSTB || BMIPS_GENERIC || ARCH_BCM_NSP || \ + ARCH_BCM_63XX help This option enables support for the AHCI SATA3 controller found on Broadcom SoC's. diff --git a/drivers/ata/ahci.h b/drivers/ata/ahci.h index 6a1515f0da40..ef356e70e6de 100644 --- a/drivers/ata/ahci.h +++ b/drivers/ata/ahci.h @@ -352,6 +352,8 @@ struct ahci_host_priv { struct clk *clks[AHCI_MAX_CLKS]; /* Optional */ struct reset_control *rsts; /* Optional */ struct regulator **target_pwrs; /* Optional */ + struct regulator *ahci_regulator;/* Optional */ + struct regulator *phy_regulator;/* Optional */ /* * If platform uses PHYs. There is a 1:1 relation between the port number and * the PHY position in this array. diff --git a/drivers/ata/ahci_brcm.c b/drivers/ata/ahci_brcm.c index f3d557777d82..fba5a3044c8a 100644 --- a/drivers/ata/ahci_brcm.c +++ b/drivers/ata/ahci_brcm.c @@ -25,6 +25,7 @@ #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> +#include <linux/reset.h> #include <linux/string.h> #include "ahci.h" @@ -94,6 +95,7 @@ struct brcm_ahci_priv { u32 port_mask; u32 quirks; enum brcm_ahci_version version; + struct reset_control *rcdev; }; static inline u32 brcm_sata_readreg(void __iomem *addr) @@ -381,6 +383,7 @@ static struct scsi_host_template ahci_platform_sht = { static const struct of_device_id ahci_of_match[] = { {.compatible = "brcm,bcm7425-ahci", .data = (void *)BRCM_SATA_BCM7425}, {.compatible = "brcm,bcm7445-ahci", .data = (void *)BRCM_SATA_BCM7445}, + {.compatible = "brcm,bcm63138-ahci", .data = (void *)BRCM_SATA_BCM7445}, {.compatible = "brcm,bcm-nsp-ahci", .data = (void *)BRCM_SATA_NSP}, {}, }; @@ -411,6 +414,11 @@ static int brcm_ahci_probe(struct platform_device *pdev) if (IS_ERR(priv->top_ctrl)) return PTR_ERR(priv->top_ctrl); + /* Reset is optional depending on platform */ + priv->rcdev = devm_reset_control_get(&pdev->dev, "ahci"); + if (!IS_ERR_OR_NULL(priv->rcdev)) + reset_control_deassert(priv->rcdev); + if ((priv->version == BRCM_SATA_BCM7425) || (priv->version == BRCM_SATA_NSP)) { priv->quirks |= BRCM_AHCI_QUIRK_NO_NCQ; diff --git a/drivers/ata/ahci_platform.c b/drivers/ata/ahci_platform.c index 46f0bd75eff7..cf1e0e18a7a9 100644 --- a/drivers/ata/ahci_platform.c +++ b/drivers/ata/ahci_platform.c @@ -33,6 +33,13 @@ static const struct ata_port_info ahci_port_info = { .port_ops = &ahci_platform_ops, }; +static const struct ata_port_info ahci_port_info_nolpm = { + .flags = AHCI_FLAG_COMMON | ATA_FLAG_NO_LPM, + .pio_mask = ATA_PIO4, + .udma_mask = ATA_UDMA6, + .port_ops = &ahci_platform_ops, +}; + static struct scsi_host_template ahci_platform_sht = { AHCI_SHT(DRV_NAME), }; @@ -41,6 +48,7 @@ static int ahci_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct ahci_host_priv *hpriv; + const struct ata_port_info *port; int rc; hpriv = ahci_platform_get_resources(pdev, @@ -58,7 +66,11 @@ static int ahci_probe(struct platform_device *pdev) if (of_device_is_compatible(dev->of_node, "hisilicon,hisi-ahci")) hpriv->flags |= AHCI_HFLAG_NO_FBS | AHCI_HFLAG_NO_NCQ; - rc = ahci_platform_init_host(pdev, hpriv, &ahci_port_info, + port = acpi_device_get_match_data(dev); + if (!port) + port = &ahci_port_info; + + rc = ahci_platform_init_host(pdev, hpriv, port, &ahci_platform_sht); if (rc) goto disable_resources; @@ -85,6 +97,7 @@ static const struct of_device_id ahci_of_match[] = { MODULE_DEVICE_TABLE(of, ahci_of_match); static const struct acpi_device_id ahci_acpi_match[] = { + { "APMC0D33", (unsigned long)&ahci_port_info_nolpm }, { ACPI_DEVICE_CLASS(PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff) }, {}, }; diff --git a/drivers/ata/ahci_sunxi.c b/drivers/ata/ahci_sunxi.c index 631610b72aa5..911710643305 100644 --- a/drivers/ata/ahci_sunxi.c +++ b/drivers/ata/ahci_sunxi.c @@ -181,7 +181,7 @@ static int ahci_sunxi_probe(struct platform_device *pdev) struct ahci_host_priv *hpriv; int rc; - hpriv = ahci_platform_get_resources(pdev, 0); + hpriv = ahci_platform_get_resources(pdev, AHCI_PLATFORM_GET_RESETS); if (IS_ERR(hpriv)) return PTR_ERR(hpriv); @@ -250,6 +250,7 @@ static SIMPLE_DEV_PM_OPS(ahci_sunxi_pm_ops, ahci_platform_suspend, static const struct of_device_id ahci_sunxi_of_match[] = { { .compatible = "allwinner,sun4i-a10-ahci", }, + { .compatible = "allwinner,sun8i-r40-ahci", }, { }, }; MODULE_DEVICE_TABLE(of, ahci_sunxi_of_match); diff --git a/drivers/ata/libahci_platform.c b/drivers/ata/libahci_platform.c index c92c10d55374..4b900fc659f7 100644 --- a/drivers/ata/libahci_platform.c +++ b/drivers/ata/libahci_platform.c @@ -139,7 +139,7 @@ EXPORT_SYMBOL_GPL(ahci_platform_disable_clks); * ahci_platform_enable_regulators - Enable regulators * @hpriv: host private area to store config values * - * This function enables all the regulators found in + * This function enables all the regulators found in controller and * hpriv->target_pwrs, if any. If a regulator fails to be enabled, it * disables all the regulators already enabled in reverse order and * returns an error. @@ -151,6 +151,18 @@ int ahci_platform_enable_regulators(struct ahci_host_priv *hpriv) { int rc, i; + if (hpriv->ahci_regulator) { + rc = regulator_enable(hpriv->ahci_regulator); + if (rc) + return rc; + } + + if (hpriv->phy_regulator) { + rc = regulator_enable(hpriv->phy_regulator); + if (rc) + goto disable_ahci_pwrs; + } + for (i = 0; i < hpriv->nports; i++) { if (!hpriv->target_pwrs[i]) continue; @@ -167,6 +179,11 @@ disable_target_pwrs: if (hpriv->target_pwrs[i]) regulator_disable(hpriv->target_pwrs[i]); + if (hpriv->phy_regulator) + regulator_disable(hpriv->phy_regulator); +disable_ahci_pwrs: + if (hpriv->ahci_regulator) + regulator_disable(hpriv->ahci_regulator); return rc; } EXPORT_SYMBOL_GPL(ahci_platform_enable_regulators); @@ -175,7 +192,8 @@ EXPORT_SYMBOL_GPL(ahci_platform_enable_regulators); * ahci_platform_disable_regulators - Disable regulators * @hpriv: host private area to store config values * - * This function disables all regulators found in hpriv->target_pwrs. + * This function disables all regulators found in hpriv->target_pwrs and + * AHCI controller. */ void ahci_platform_disable_regulators(struct ahci_host_priv *hpriv) { @@ -186,6 +204,11 @@ void ahci_platform_disable_regulators(struct ahci_host_priv *hpriv) continue; regulator_disable(hpriv->target_pwrs[i]); } + + if (hpriv->ahci_regulator) + regulator_disable(hpriv->ahci_regulator); + if (hpriv->phy_regulator) + regulator_disable(hpriv->phy_regulator); } EXPORT_SYMBOL_GPL(ahci_platform_disable_regulators); /** @@ -303,8 +326,8 @@ static int ahci_platform_get_phy(struct ahci_host_priv *hpriv, u32 port, /* No PHY support. Check if PHY is required. */ if (of_find_property(node, "phys", NULL)) { dev_err(dev, - "couldn't get PHY in node %s: ENOSYS\n", - node->name); + "couldn't get PHY in node %pOFn: ENOSYS\n", + node); break; } /* fall through */ @@ -316,8 +339,8 @@ static int ahci_platform_get_phy(struct ahci_host_priv *hpriv, u32 port, default: dev_err(dev, - "couldn't get PHY in node %s: %d\n", - node->name, rc); + "couldn't get PHY in node %pOFn: %d\n", + node, rc); break; } @@ -351,6 +374,7 @@ static int ahci_platform_get_regulator(struct ahci_host_priv *hpriv, u32 port, * * 1) mmio registers (IORESOURCE_MEM 0, mandatory) * 2) regulator for controlling the targets power (optional) + * regulator for controlling the AHCI controller (optional) * 3) 0 - AHCI_MAX_CLKS clocks, as specified in the devs devicetree node, * or for non devicetree enabled platforms a single clock * 4) resets, if flags has AHCI_PLATFORM_GET_RESETS (optional) @@ -408,6 +432,24 @@ struct ahci_host_priv *ahci_platform_get_resources(struct platform_device *pdev, hpriv->clks[i] = clk; } + hpriv->ahci_regulator = devm_regulator_get_optional(dev, "ahci"); + if (IS_ERR(hpriv->ahci_regulator)) { + rc = PTR_ERR(hpriv->ahci_regulator); + if (rc == -EPROBE_DEFER) + goto err_out; + rc = 0; + hpriv->ahci_regulator = NULL; + } + + hpriv->phy_regulator = devm_regulator_get_optional(dev, "phy"); + if (IS_ERR(hpriv->phy_regulator)) { + rc = PTR_ERR(hpriv->phy_regulator); + if (rc == -EPROBE_DEFER) + goto err_out; + rc = 0; + hpriv->phy_regulator = NULL; + } + if (flags & AHCI_PLATFORM_GET_RESETS) { hpriv->rsts = devm_reset_control_array_get_optional_shared(dev); if (IS_ERR(hpriv->rsts)) { diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index 1984fc78c750..3d4887d0e84a 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -639,8 +639,8 @@ int ata_cmd_ioctl(struct scsi_device *scsidev, void __user *arg) if (args[0] == ATA_CMD_SMART) { /* hack -- ide driver does this too */ scsi_cmd[6] = args[3]; scsi_cmd[8] = args[1]; - scsi_cmd[10] = 0x4f; - scsi_cmd[12] = 0xc2; + scsi_cmd[10] = ATA_SMART_LBAM_PASS; + scsi_cmd[12] = ATA_SMART_LBAH_PASS; } else { scsi_cmd[6] = args[1]; } diff --git a/drivers/ata/pata_atiixp.c b/drivers/ata/pata_atiixp.c index 4d49fd3c927b..843bb200a1ee 100644 --- a/drivers/ata/pata_atiixp.c +++ b/drivers/ata/pata_atiixp.c @@ -279,7 +279,7 @@ static int atiixp_init_one(struct pci_dev *pdev, const struct pci_device_id *id) const struct ata_port_info *ppi[] = { &info, &info }; /* SB600 doesn't have secondary port wired */ - if((pdev->device == PCI_DEVICE_ID_ATI_IXP600_IDE)) + if (pdev->device == PCI_DEVICE_ID_ATI_IXP600_IDE) ppi[1] = &ata_dummy_port_info; return ata_pci_bmdma_init_one(pdev, ppi, &atiixp_sht, NULL, diff --git a/drivers/ata/pata_ep93xx.c b/drivers/ata/pata_ep93xx.c index 0a550190955a..cc6d06c1b2c7 100644 --- a/drivers/ata/pata_ep93xx.c +++ b/drivers/ata/pata_ep93xx.c @@ -659,7 +659,7 @@ static void ep93xx_pata_dma_init(struct ep93xx_pata_data *drv_data) * start of new transfer. */ drv_data->dma_rx_data.port = EP93XX_DMA_IDE; - drv_data->dma_rx_data.direction = DMA_FROM_DEVICE; + drv_data->dma_rx_data.direction = DMA_DEV_TO_MEM; drv_data->dma_rx_data.name = "ep93xx-pata-rx"; drv_data->dma_rx_channel = dma_request_channel(mask, ep93xx_pata_dma_filter, &drv_data->dma_rx_data); @@ -667,7 +667,7 @@ static void ep93xx_pata_dma_init(struct ep93xx_pata_data *drv_data) return; drv_data->dma_tx_data.port = EP93XX_DMA_IDE; - drv_data->dma_tx_data.direction = DMA_TO_DEVICE; + drv_data->dma_tx_data.direction = DMA_MEM_TO_DEV; drv_data->dma_tx_data.name = "ep93xx-pata-tx"; drv_data->dma_tx_channel = dma_request_channel(mask, ep93xx_pata_dma_filter, &drv_data->dma_tx_data); @@ -678,7 +678,7 @@ static void ep93xx_pata_dma_init(struct ep93xx_pata_data *drv_data) /* Configure receive channel direction and source address */ memset(&conf, 0, sizeof(conf)); - conf.direction = DMA_FROM_DEVICE; + conf.direction = DMA_DEV_TO_MEM; conf.src_addr = drv_data->udma_in_phys; conf.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; if (dmaengine_slave_config(drv_data->dma_rx_channel, &conf)) { @@ -689,7 +689,7 @@ static void ep93xx_pata_dma_init(struct ep93xx_pata_data *drv_data) /* Configure transmit channel direction and destination address */ memset(&conf, 0, sizeof(conf)); - conf.direction = DMA_TO_DEVICE; + conf.direction = DMA_MEM_TO_DEV; conf.dst_addr = drv_data->udma_out_phys; conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; if (dmaengine_slave_config(drv_data->dma_tx_channel, &conf)) { diff --git a/drivers/base/dd.c b/drivers/base/dd.c index edfc9f0b1180..169412ee4ae8 100644 --- a/drivers/base/dd.c +++ b/drivers/base/dd.c @@ -480,9 +480,11 @@ re_probe: if (ret) goto pinctrl_bind_failed; - ret = dma_configure(dev); - if (ret) - goto dma_failed; + if (dev->bus->dma_configure) { + ret = dev->bus->dma_configure(dev); + if (ret) + goto dma_failed; + } if (driver_sysfs_add(dev)) { printk(KERN_ERR "%s: driver_sysfs_add(%s) failed\n", @@ -537,7 +539,7 @@ re_probe: goto done; probe_failed: - dma_deconfigure(dev); + arch_teardown_dma_ops(dev); dma_failed: if (dev->bus) blocking_notifier_call_chain(&dev->bus->p->bus_notifier, @@ -966,7 +968,7 @@ static void __device_release_driver(struct device *dev, struct device *parent) drv->remove(dev); device_links_driver_cleanup(dev); - dma_deconfigure(dev); + arch_teardown_dma_ops(dev); devres_release_all(dev); dev->driver = NULL; diff --git a/drivers/base/platform.c b/drivers/base/platform.c index dff82a3c2caa..23cf4427f425 100644 --- a/drivers/base/platform.c +++ b/drivers/base/platform.c @@ -1180,7 +1180,7 @@ int __init platform_bus_init(void) } #ifndef ARCH_HAS_DMA_GET_REQUIRED_MASK -u64 dma_get_required_mask(struct device *dev) +static u64 dma_default_get_required_mask(struct device *dev) { u32 low_totalram = ((max_pfn - 1) << PAGE_SHIFT); u32 high_totalram = ((max_pfn - 1) >> (32 - PAGE_SHIFT)); @@ -1198,6 +1198,15 @@ u64 dma_get_required_mask(struct device *dev) } return mask; } + +u64 dma_get_required_mask(struct device *dev) +{ + const struct dma_map_ops *ops = get_dma_ops(dev); + + if (ops->get_required_mask) + return ops->get_required_mask(dev); + return dma_default_get_required_mask(dev); +} EXPORT_SYMBOL_GPL(dma_get_required_mask); #endif diff --git a/drivers/base/regmap/internal.h b/drivers/base/regmap/internal.h index a6bf34d6394e..a98fced9bff8 100644 --- a/drivers/base/regmap/internal.h +++ b/drivers/base/regmap/internal.h @@ -94,11 +94,13 @@ struct regmap { bool (*readable_reg)(struct device *dev, unsigned int reg); bool (*volatile_reg)(struct device *dev, unsigned int reg); bool (*precious_reg)(struct device *dev, unsigned int reg); + bool (*writeable_noinc_reg)(struct device *dev, unsigned int reg); bool (*readable_noinc_reg)(struct device *dev, unsigned int reg); const struct regmap_access_table *wr_table; const struct regmap_access_table *rd_table; const struct regmap_access_table *volatile_table; const struct regmap_access_table *precious_table; + const struct regmap_access_table *wr_noinc_table; const struct regmap_access_table *rd_noinc_table; int (*reg_read)(void *context, unsigned int reg, unsigned int *val); @@ -149,7 +151,7 @@ struct regmap { /* if set, converts bulk read to single read */ bool use_single_read; - /* if set, converts bulk read to single read */ + /* if set, converts bulk write to single write */ bool use_single_write; /* if set, the device supports multi write mode */ bool can_multi_write; @@ -183,6 +185,7 @@ bool regmap_writeable(struct regmap *map, unsigned int reg); bool regmap_readable(struct regmap *map, unsigned int reg); bool regmap_volatile(struct regmap *map, unsigned int reg); bool regmap_precious(struct regmap *map, unsigned int reg); +bool regmap_writeable_noinc(struct regmap *map, unsigned int reg); bool regmap_readable_noinc(struct regmap *map, unsigned int reg); int _regmap_write(struct regmap *map, unsigned int reg, diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c index 0360a90ad6b6..4f822e087def 100644 --- a/drivers/base/regmap/regmap.c +++ b/drivers/base/regmap/regmap.c @@ -35,6 +35,16 @@ */ #undef LOG_DEVICE +#ifdef LOG_DEVICE +static inline bool regmap_should_log(struct regmap *map) +{ + return (map->dev && strcmp(dev_name(map->dev), LOG_DEVICE) == 0); +} +#else +static inline bool regmap_should_log(struct regmap *map) { return false; } +#endif + + static int _regmap_update_bits(struct regmap *map, unsigned int reg, unsigned int mask, unsigned int val, bool *change, bool force_write); @@ -168,6 +178,17 @@ bool regmap_precious(struct regmap *map, unsigned int reg) return false; } +bool regmap_writeable_noinc(struct regmap *map, unsigned int reg) +{ + if (map->writeable_noinc_reg) + return map->writeable_noinc_reg(map->dev, reg); + + if (map->wr_noinc_table) + return regmap_check_range_table(map, reg, map->wr_noinc_table); + + return true; +} + bool regmap_readable_noinc(struct regmap *map, unsigned int reg) { if (map->readable_noinc_reg) @@ -762,8 +783,8 @@ struct regmap *__regmap_init(struct device *dev, map->reg_stride_order = ilog2(map->reg_stride); else map->reg_stride_order = -1; - map->use_single_read = config->use_single_rw || !bus || !bus->read; - map->use_single_write = config->use_single_rw || !bus || !bus->write; + map->use_single_read = config->use_single_read || !bus || !bus->read; + map->use_single_write = config->use_single_write || !bus || !bus->write; map->can_multi_write = config->can_multi_write && bus && bus->write; if (bus) { map->max_raw_read = bus->max_raw_read; @@ -777,11 +798,13 @@ struct regmap *__regmap_init(struct device *dev, map->rd_table = config->rd_table; map->volatile_table = config->volatile_table; map->precious_table = config->precious_table; + map->wr_noinc_table = config->wr_noinc_table; map->rd_noinc_table = config->rd_noinc_table; map->writeable_reg = config->writeable_reg; map->readable_reg = config->readable_reg; map->volatile_reg = config->volatile_reg; map->precious_reg = config->precious_reg; + map->writeable_noinc_reg = config->writeable_noinc_reg; map->readable_noinc_reg = config->readable_noinc_reg; map->cache_type = config->cache_type; @@ -1298,6 +1321,7 @@ int regmap_reinit_cache(struct regmap *map, const struct regmap_config *config) map->readable_reg = config->readable_reg; map->volatile_reg = config->volatile_reg; map->precious_reg = config->precious_reg; + map->writeable_noinc_reg = config->writeable_noinc_reg; map->readable_noinc_reg = config->readable_noinc_reg; map->cache_type = config->cache_type; @@ -1755,10 +1779,8 @@ int _regmap_write(struct regmap *map, unsigned int reg, } } -#ifdef LOG_DEVICE - if (map->dev && strcmp(dev_name(map->dev), LOG_DEVICE) == 0) + if (regmap_should_log(map)) dev_info(map->dev, "%x <= %x\n", reg, val); -#endif trace_regmap_reg_write(map, reg, val); @@ -1898,6 +1920,69 @@ int regmap_raw_write(struct regmap *map, unsigned int reg, EXPORT_SYMBOL_GPL(regmap_raw_write); /** + * regmap_noinc_write(): Write data from a register without incrementing the + * register number + * + * @map: Register map to write to + * @reg: Register to write to + * @val: Pointer to data buffer + * @val_len: Length of output buffer in bytes. + * + * The regmap API usually assumes that bulk bus write operations will write a + * range of registers. Some devices have certain registers for which a write + * operation can write to an internal FIFO. + * + * The target register must be volatile but registers after it can be + * completely unrelated cacheable registers. + * + * This will attempt multiple writes as required to write val_len bytes. + * + * A value of zero will be returned on success, a negative errno will be + * returned in error cases. + */ +int regmap_noinc_write(struct regmap *map, unsigned int reg, + const void *val, size_t val_len) +{ + size_t write_len; + int ret; + + if (!map->bus) + return -EINVAL; + if (!map->bus->write) + return -ENOTSUPP; + if (val_len % map->format.val_bytes) + return -EINVAL; + if (!IS_ALIGNED(reg, map->reg_stride)) + return -EINVAL; + if (val_len == 0) + return -EINVAL; + + map->lock(map->lock_arg); + + if (!regmap_volatile(map, reg) || !regmap_writeable_noinc(map, reg)) { + ret = -EINVAL; + goto out_unlock; + } + + while (val_len) { + if (map->max_raw_write && map->max_raw_write < val_len) + write_len = map->max_raw_write; + else + write_len = val_len; + ret = _regmap_raw_write(map, reg, val, write_len); + if (ret) + goto out_unlock; + val = ((u8 *)val) + write_len; + val_len -= write_len; + } + +out_unlock: + map->unlock(map->lock_arg); + return ret; +} +EXPORT_SYMBOL_GPL(regmap_noinc_write); + +/** * regmap_field_update_bits_base() - Perform a read/modify/write cycle a * register field. * @@ -2450,10 +2535,8 @@ static int _regmap_read(struct regmap *map, unsigned int reg, ret = map->reg_read(context, reg, val); if (ret == 0) { -#ifdef LOG_DEVICE - if (map->dev && strcmp(dev_name(map->dev), LOG_DEVICE) == 0) + if (regmap_should_log(map)) dev_info(map->dev, "%x => %x\n", reg, *val); -#endif trace_regmap_reg_read(map, reg, *val); diff --git a/drivers/block/DAC960.c b/drivers/block/DAC960.c deleted file mode 100644 index 581312ac375f..000000000000 --- a/drivers/block/DAC960.c +++ /dev/null @@ -1,7229 +0,0 @@ -/* - - Linux Driver for Mylex DAC960/AcceleRAID/eXtremeRAID PCI RAID Controllers - - Copyright 1998-2001 by Leonard N. Zubkoff <lnz@dandelion.com> - Portions Copyright 2002 by Mylex (An IBM Business Unit) - - This program is free software; you may redistribute and/or modify it under - the terms of the GNU General Public License Version 2 as published by the - Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY, without even the implied warranty of MERCHANTABILITY - or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for complete details. - -*/ - - -#define DAC960_DriverVersion "2.5.49" -#define DAC960_DriverDate "21 Aug 2007" - - -#include <linux/compiler.h> -#include <linux/module.h> -#include <linux/types.h> -#include <linux/miscdevice.h> -#include <linux/blkdev.h> -#include <linux/bio.h> -#include <linux/completion.h> -#include <linux/delay.h> -#include <linux/genhd.h> -#include <linux/hdreg.h> -#include <linux/blkpg.h> -#include <linux/dma-mapping.h> -#include <linux/interrupt.h> -#include <linux/ioport.h> -#include <linux/mm.h> -#include <linux/slab.h> -#include <linux/mutex.h> -#include <linux/proc_fs.h> -#include <linux/seq_file.h> -#include <linux/reboot.h> -#include <linux/spinlock.h> -#include <linux/timer.h> -#include <linux/pci.h> -#include <linux/init.h> -#include <linux/jiffies.h> -#include <linux/random.h> -#include <linux/scatterlist.h> -#include <asm/io.h> -#include <linux/uaccess.h> -#include "DAC960.h" - -#define DAC960_GAM_MINOR 252 - - -static DEFINE_MUTEX(DAC960_mutex); -static DAC960_Controller_T *DAC960_Controllers[DAC960_MaxControllers]; -static int DAC960_ControllerCount; -static struct proc_dir_entry *DAC960_ProcDirectoryEntry; - -static long disk_size(DAC960_Controller_T *p, int drive_nr) -{ - if (p->FirmwareType == DAC960_V1_Controller) { - if (drive_nr >= p->LogicalDriveCount) - return 0; - return p->V1.LogicalDriveInformation[drive_nr]. - LogicalDriveSize; - } else { - DAC960_V2_LogicalDeviceInfo_T *i = - p->V2.LogicalDeviceInformation[drive_nr]; - if (i == NULL) - return 0; - return i->ConfigurableDeviceSize; - } -} - -static int DAC960_open(struct block_device *bdev, fmode_t mode) -{ - struct gendisk *disk = bdev->bd_disk; - DAC960_Controller_T *p = disk->queue->queuedata; - int drive_nr = (long)disk->private_data; - int ret = -ENXIO; - - mutex_lock(&DAC960_mutex); - if (p->FirmwareType == DAC960_V1_Controller) { - if (p->V1.LogicalDriveInformation[drive_nr]. - LogicalDriveState == DAC960_V1_LogicalDrive_Offline) - goto out; - } else { - DAC960_V2_LogicalDeviceInfo_T *i = - p->V2.LogicalDeviceInformation[drive_nr]; - if (!i || i->LogicalDeviceState == DAC960_V2_LogicalDevice_Offline) - goto out; - } - - check_disk_change(bdev); - - if (!get_capacity(p->disks[drive_nr])) - goto out; - ret = 0; -out: - mutex_unlock(&DAC960_mutex); - return ret; -} - -static int DAC960_getgeo(struct block_device *bdev, struct hd_geometry *geo) -{ - struct gendisk *disk = bdev->bd_disk; - DAC960_Controller_T *p = disk->queue->queuedata; - int drive_nr = (long)disk->private_data; - - if (p->FirmwareType == DAC960_V1_Controller) { - geo->heads = p->V1.GeometryTranslationHeads; - geo->sectors = p->V1.GeometryTranslationSectors; - geo->cylinders = p->V1.LogicalDriveInformation[drive_nr]. - LogicalDriveSize / (geo->heads * geo->sectors); - } else { - DAC960_V2_LogicalDeviceInfo_T *i = - p->V2.LogicalDeviceInformation[drive_nr]; - switch (i->DriveGeometry) { - case DAC960_V2_Geometry_128_32: - geo->heads = 128; - geo->sectors = 32; - break; - case DAC960_V2_Geometry_255_63: - geo->heads = 255; - geo->sectors = 63; - break; - default: - DAC960_Error("Illegal Logical Device Geometry %d\n", - p, i->DriveGeometry); - return -EINVAL; - } - - geo->cylinders = i->ConfigurableDeviceSize / - (geo->heads * geo->sectors); - } - - return 0; -} - -static unsigned int DAC960_check_events(struct gendisk *disk, - unsigned int clearing) -{ - DAC960_Controller_T *p = disk->queue->queuedata; - int drive_nr = (long)disk->private_data; - - if (!p->LogicalDriveInitiallyAccessible[drive_nr]) - return DISK_EVENT_MEDIA_CHANGE; - return 0; -} - -static int DAC960_revalidate_disk(struct gendisk *disk) -{ - DAC960_Controller_T *p = disk->queue->queuedata; - int unit = (long)disk->private_data; - - set_capacity(disk, disk_size(p, unit)); - return 0; -} - -static const struct block_device_operations DAC960_BlockDeviceOperations = { - .owner = THIS_MODULE, - .open = DAC960_open, - .getgeo = DAC960_getgeo, - .check_events = DAC960_check_events, - .revalidate_disk = DAC960_revalidate_disk, -}; - - -/* - DAC960_AnnounceDriver announces the Driver Version and Date, Author's Name, - Copyright Notice, and Electronic Mail Address. -*/ - -static void DAC960_AnnounceDriver(DAC960_Controller_T *Controller) -{ - DAC960_Announce("***** DAC960 RAID Driver Version " - DAC960_DriverVersion " of " - DAC960_DriverDate " *****\n", Controller); - DAC960_Announce("Copyright 1998-2001 by Leonard N. Zubkoff " - "<lnz@dandelion.com>\n", Controller); -} - - -/* - DAC960_Failure prints a standardized error message, and then returns false. -*/ - -static bool DAC960_Failure(DAC960_Controller_T *Controller, - unsigned char *ErrorMessage) -{ - DAC960_Error("While configuring DAC960 PCI RAID Controller at\n", - Controller); - if (Controller->IO_Address == 0) - DAC960_Error("PCI Bus %d Device %d Function %d I/O Address N/A " - "PCI Address 0x%X\n", Controller, - Controller->Bus, Controller->Device, - Controller->Function, Controller->PCI_Address); - else DAC960_Error("PCI Bus %d Device %d Function %d I/O Address " - "0x%X PCI Address 0x%X\n", Controller, - Controller->Bus, Controller->Device, - Controller->Function, Controller->IO_Address, - Controller->PCI_Address); - DAC960_Error("%s FAILED - DETACHING\n", Controller, ErrorMessage); - return false; -} - -/* - init_dma_loaf() and slice_dma_loaf() are helper functions for - aggregating the dma-mapped memory for a well-known collection of - data structures that are of different lengths. - - These routines don't guarantee any alignment. The caller must - include any space needed for alignment in the sizes of the structures - that are passed in. - */ - -static bool init_dma_loaf(struct pci_dev *dev, struct dma_loaf *loaf, - size_t len) -{ - void *cpu_addr; - dma_addr_t dma_handle; - - cpu_addr = pci_alloc_consistent(dev, len, &dma_handle); - if (cpu_addr == NULL) - return false; - - loaf->cpu_free = loaf->cpu_base = cpu_addr; - loaf->dma_free =loaf->dma_base = dma_handle; - loaf->length = len; - memset(cpu_addr, 0, len); - return true; -} - -static void *slice_dma_loaf(struct dma_loaf *loaf, size_t len, - dma_addr_t *dma_handle) -{ - void *cpu_end = loaf->cpu_free + len; - void *cpu_addr = loaf->cpu_free; - - BUG_ON(cpu_end > loaf->cpu_base + loaf->length); - *dma_handle = loaf->dma_free; - loaf->cpu_free = cpu_end; - loaf->dma_free += len; - return cpu_addr; -} - -static void free_dma_loaf(struct pci_dev *dev, struct dma_loaf *loaf_handle) -{ - if (loaf_handle->cpu_base != NULL) - pci_free_consistent(dev, loaf_handle->length, - loaf_handle->cpu_base, loaf_handle->dma_base); -} - - -/* - DAC960_CreateAuxiliaryStructures allocates and initializes the auxiliary - data structures for Controller. It returns true on success and false on - failure. -*/ - -static bool DAC960_CreateAuxiliaryStructures(DAC960_Controller_T *Controller) -{ - int CommandAllocationLength, CommandAllocationGroupSize; - int CommandsRemaining = 0, CommandIdentifier, CommandGroupByteCount; - void *AllocationPointer = NULL; - void *ScatterGatherCPU = NULL; - dma_addr_t ScatterGatherDMA; - struct dma_pool *ScatterGatherPool; - void *RequestSenseCPU = NULL; - dma_addr_t RequestSenseDMA; - struct dma_pool *RequestSensePool = NULL; - - if (Controller->FirmwareType == DAC960_V1_Controller) - { - CommandAllocationLength = offsetof(DAC960_Command_T, V1.EndMarker); - CommandAllocationGroupSize = DAC960_V1_CommandAllocationGroupSize; - ScatterGatherPool = dma_pool_create("DAC960_V1_ScatterGather", - &Controller->PCIDevice->dev, - DAC960_V1_ScatterGatherLimit * sizeof(DAC960_V1_ScatterGatherSegment_T), - sizeof(DAC960_V1_ScatterGatherSegment_T), 0); - if (ScatterGatherPool == NULL) - return DAC960_Failure(Controller, - "AUXILIARY STRUCTURE CREATION (SG)"); - Controller->ScatterGatherPool = ScatterGatherPool; - } - else - { - CommandAllocationLength = offsetof(DAC960_Command_T, V2.EndMarker); - CommandAllocationGroupSize = DAC960_V2_CommandAllocationGroupSize; - ScatterGatherPool = dma_pool_create("DAC960_V2_ScatterGather", - &Controller->PCIDevice->dev, - DAC960_V2_ScatterGatherLimit * sizeof(DAC960_V2_ScatterGatherSegment_T), - sizeof(DAC960_V2_ScatterGatherSegment_T), 0); - if (ScatterGatherPool == NULL) - return DAC960_Failure(Controller, - "AUXILIARY STRUCTURE CREATION (SG)"); - RequestSensePool = dma_pool_create("DAC960_V2_RequestSense", - &Controller->PCIDevice->dev, sizeof(DAC960_SCSI_RequestSense_T), - sizeof(int), 0); - if (RequestSensePool == NULL) { - dma_pool_destroy(ScatterGatherPool); - return DAC960_Failure(Controller, - "AUXILIARY STRUCTURE CREATION (SG)"); - } - Controller->ScatterGatherPool = ScatterGatherPool; - Controller->V2.RequestSensePool = RequestSensePool; - } - Controller->CommandAllocationGroupSize = CommandAllocationGroupSize; - Controller->FreeCommands = NULL; - for (CommandIdentifier = 1; - CommandIdentifier <= Controller->DriverQueueDepth; - CommandIdentifier++) - { - DAC960_Command_T *Command; - if (--CommandsRemaining <= 0) - { - CommandsRemaining = - Controller->DriverQueueDepth - CommandIdentifier + 1; - if (CommandsRemaining > CommandAllocationGroupSize) - CommandsRemaining = CommandAllocationGroupSize; - CommandGroupByteCount = - CommandsRemaining * CommandAllocationLength; - AllocationPointer = kzalloc(CommandGroupByteCount, GFP_ATOMIC); - if (AllocationPointer == NULL) - return DAC960_Failure(Controller, - "AUXILIARY STRUCTURE CREATION"); - } - Command = (DAC960_Command_T *) AllocationPointer; - AllocationPointer += CommandAllocationLength; - Command->CommandIdentifier = CommandIdentifier; - Command->Controller = Controller; - Command->Next = Controller->FreeCommands; - Controller->FreeCommands = Command; - Controller->Commands[CommandIdentifier-1] = Command; - ScatterGatherCPU = dma_pool_alloc(ScatterGatherPool, GFP_ATOMIC, - &ScatterGatherDMA); - if (ScatterGatherCPU == NULL) - return DAC960_Failure(Controller, "AUXILIARY STRUCTURE CREATION"); - - if (RequestSensePool != NULL) { - RequestSenseCPU = dma_pool_alloc(RequestSensePool, GFP_ATOMIC, - &RequestSenseDMA); - if (RequestSenseCPU == NULL) { - dma_pool_free(ScatterGatherPool, ScatterGatherCPU, - ScatterGatherDMA); - return DAC960_Failure(Controller, - "AUXILIARY STRUCTURE CREATION"); - } - } - if (Controller->FirmwareType == DAC960_V1_Controller) { - Command->cmd_sglist = Command->V1.ScatterList; - Command->V1.ScatterGatherList = - (DAC960_V1_ScatterGatherSegment_T *)ScatterGatherCPU; - Command->V1.ScatterGatherListDMA = ScatterGatherDMA; - sg_init_table(Command->cmd_sglist, DAC960_V1_ScatterGatherLimit); - } else { - Command->cmd_sglist = Command->V2.ScatterList; - Command->V2.ScatterGatherList = - (DAC960_V2_ScatterGatherSegment_T *)ScatterGatherCPU; - Command->V2.ScatterGatherListDMA = ScatterGatherDMA; - Command->V2.RequestSense = - (DAC960_SCSI_RequestSense_T *)RequestSenseCPU; - Command->V2.RequestSenseDMA = RequestSenseDMA; - sg_init_table(Command->cmd_sglist, DAC960_V2_ScatterGatherLimit); - } - } - return true; -} - - -/* - DAC960_DestroyAuxiliaryStructures deallocates the auxiliary data - structures for Controller. -*/ - -static void DAC960_DestroyAuxiliaryStructures(DAC960_Controller_T *Controller) -{ - int i; - struct dma_pool *ScatterGatherPool = Controller->ScatterGatherPool; - struct dma_pool *RequestSensePool = NULL; - void *ScatterGatherCPU; - dma_addr_t ScatterGatherDMA; - void *RequestSenseCPU; - dma_addr_t RequestSenseDMA; - DAC960_Command_T *CommandGroup = NULL; - - - if (Controller->FirmwareType == DAC960_V2_Controller) - RequestSensePool = Controller->V2.RequestSensePool; - - Controller->FreeCommands = NULL; - for (i = 0; i < Controller->DriverQueueDepth; i++) - { - DAC960_Command_T *Command = Controller->Commands[i]; - - if (Command == NULL) - continue; - - if (Controller->FirmwareType == DAC960_V1_Controller) { - ScatterGatherCPU = (void *)Command->V1.ScatterGatherList; - ScatterGatherDMA = Command->V1.ScatterGatherListDMA; - RequestSenseCPU = NULL; - RequestSenseDMA = (dma_addr_t)0; - } else { - ScatterGatherCPU = (void *)Command->V2.ScatterGatherList; - ScatterGatherDMA = Command->V2.ScatterGatherListDMA; - RequestSenseCPU = (void *)Command->V2.RequestSense; - RequestSenseDMA = Command->V2.RequestSenseDMA; - } - if (ScatterGatherCPU != NULL) - dma_pool_free(ScatterGatherPool, ScatterGatherCPU, ScatterGatherDMA); - if (RequestSenseCPU != NULL) - dma_pool_free(RequestSensePool, RequestSenseCPU, RequestSenseDMA); - - if ((Command->CommandIdentifier - % Controller->CommandAllocationGroupSize) == 1) { - /* - * We can't free the group of commands until all of the - * request sense and scatter gather dma structures are free. - * Remember the beginning of the group, but don't free it - * until we've reached the beginning of the next group. - */ - kfree(CommandGroup); - CommandGroup = Command; - } - Controller->Commands[i] = NULL; - } - kfree(CommandGroup); - - if (Controller->CombinedStatusBuffer != NULL) - { - kfree(Controller->CombinedStatusBuffer); - Controller->CombinedStatusBuffer = NULL; - Controller->CurrentStatusBuffer = NULL; - } - - dma_pool_destroy(ScatterGatherPool); - if (Controller->FirmwareType == DAC960_V1_Controller) - return; - - dma_pool_destroy(RequestSensePool); - - for (i = 0; i < DAC960_MaxLogicalDrives; i++) { - kfree(Controller->V2.LogicalDeviceInformation[i]); - Controller->V2.LogicalDeviceInformation[i] = NULL; - } - - for (i = 0; i < DAC960_V2_MaxPhysicalDevices; i++) - { - kfree(Controller->V2.PhysicalDeviceInformation[i]); - Controller->V2.PhysicalDeviceInformation[i] = NULL; - kfree(Controller->V2.InquiryUnitSerialNumber[i]); - Controller->V2.InquiryUnitSerialNumber[i] = NULL; - } -} - - -/* - DAC960_V1_ClearCommand clears critical fields of Command for DAC960 V1 - Firmware Controllers. -*/ - -static inline void DAC960_V1_ClearCommand(DAC960_Command_T *Command) -{ - DAC960_V1_CommandMailbox_T *CommandMailbox = &Command->V1.CommandMailbox; - memset(CommandMailbox, 0, sizeof(DAC960_V1_CommandMailbox_T)); - Command->V1.CommandStatus = 0; -} - - -/* - DAC960_V2_ClearCommand clears critical fields of Command for DAC960 V2 - Firmware Controllers. -*/ - -static inline void DAC960_V2_ClearCommand(DAC960_Command_T *Command) -{ - DAC960_V2_CommandMailbox_T *CommandMailbox = &Command->V2.CommandMailbox; - memset(CommandMailbox, 0, sizeof(DAC960_V2_CommandMailbox_T)); - Command->V2.CommandStatus = 0; -} - - -/* - DAC960_AllocateCommand allocates a Command structure from Controller's - free list. During driver initialization, a special initialization command - has been placed on the free list to guarantee that command allocation can - never fail. -*/ - -static inline DAC960_Command_T *DAC960_AllocateCommand(DAC960_Controller_T - *Controller) -{ - DAC960_Command_T *Command = Controller->FreeCommands; - if (Command == NULL) return NULL; - Controller->FreeCommands = Command->Next; - Command->Next = NULL; - return Command; -} - - -/* - DAC960_DeallocateCommand deallocates Command, returning it to Controller's - free list. -*/ - -static inline void DAC960_DeallocateCommand(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - - Command->Request = NULL; - Command->Next = Controller->FreeCommands; - Controller->FreeCommands = Command; -} - - -/* - DAC960_WaitForCommand waits for a wake_up on Controller's Command Wait Queue. -*/ - -static void DAC960_WaitForCommand(DAC960_Controller_T *Controller) -{ - spin_unlock_irq(&Controller->queue_lock); - __wait_event(Controller->CommandWaitQueue, Controller->FreeCommands); - spin_lock_irq(&Controller->queue_lock); -} - -/* - DAC960_GEM_QueueCommand queues Command for DAC960 GEM Series Controllers. -*/ - -static void DAC960_GEM_QueueCommand(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - DAC960_V2_CommandMailbox_T *CommandMailbox = &Command->V2.CommandMailbox; - DAC960_V2_CommandMailbox_T *NextCommandMailbox = - Controller->V2.NextCommandMailbox; - - CommandMailbox->Common.CommandIdentifier = Command->CommandIdentifier; - DAC960_GEM_WriteCommandMailbox(NextCommandMailbox, CommandMailbox); - - if (Controller->V2.PreviousCommandMailbox1->Words[0] == 0 || - Controller->V2.PreviousCommandMailbox2->Words[0] == 0) - DAC960_GEM_MemoryMailboxNewCommand(ControllerBaseAddress); - - Controller->V2.PreviousCommandMailbox2 = - Controller->V2.PreviousCommandMailbox1; - Controller->V2.PreviousCommandMailbox1 = NextCommandMailbox; - - if (++NextCommandMailbox > Controller->V2.LastCommandMailbox) - NextCommandMailbox = Controller->V2.FirstCommandMailbox; - - Controller->V2.NextCommandMailbox = NextCommandMailbox; -} - -/* - DAC960_BA_QueueCommand queues Command for DAC960 BA Series Controllers. -*/ - -static void DAC960_BA_QueueCommand(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - DAC960_V2_CommandMailbox_T *CommandMailbox = &Command->V2.CommandMailbox; - DAC960_V2_CommandMailbox_T *NextCommandMailbox = - Controller->V2.NextCommandMailbox; - CommandMailbox->Common.CommandIdentifier = Command->CommandIdentifier; - DAC960_BA_WriteCommandMailbox(NextCommandMailbox, CommandMailbox); - if (Controller->V2.PreviousCommandMailbox1->Words[0] == 0 || - Controller->V2.PreviousCommandMailbox2->Words[0] == 0) - DAC960_BA_MemoryMailboxNewCommand(ControllerBaseAddress); - Controller->V2.PreviousCommandMailbox2 = - Controller->V2.PreviousCommandMailbox1; - Controller->V2.PreviousCommandMailbox1 = NextCommandMailbox; - if (++NextCommandMailbox > Controller->V2.LastCommandMailbox) - NextCommandMailbox = Controller->V2.FirstCommandMailbox; - Controller->V2.NextCommandMailbox = NextCommandMailbox; -} - - -/* - DAC960_LP_QueueCommand queues Command for DAC960 LP Series Controllers. -*/ - -static void DAC960_LP_QueueCommand(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - DAC960_V2_CommandMailbox_T *CommandMailbox = &Command->V2.CommandMailbox; - DAC960_V2_CommandMailbox_T *NextCommandMailbox = - Controller->V2.NextCommandMailbox; - CommandMailbox->Common.CommandIdentifier = Command->CommandIdentifier; - DAC960_LP_WriteCommandMailbox(NextCommandMailbox, CommandMailbox); - if (Controller->V2.PreviousCommandMailbox1->Words[0] == 0 || - Controller->V2.PreviousCommandMailbox2->Words[0] == 0) - DAC960_LP_MemoryMailboxNewCommand(ControllerBaseAddress); - Controller->V2.PreviousCommandMailbox2 = - Controller->V2.PreviousCommandMailbox1; - Controller->V2.PreviousCommandMailbox1 = NextCommandMailbox; - if (++NextCommandMailbox > Controller->V2.LastCommandMailbox) - NextCommandMailbox = Controller->V2.FirstCommandMailbox; - Controller->V2.NextCommandMailbox = NextCommandMailbox; -} - - -/* - DAC960_LA_QueueCommandDualMode queues Command for DAC960 LA Series - Controllers with Dual Mode Firmware. -*/ - -static void DAC960_LA_QueueCommandDualMode(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - DAC960_V1_CommandMailbox_T *CommandMailbox = &Command->V1.CommandMailbox; - DAC960_V1_CommandMailbox_T *NextCommandMailbox = - Controller->V1.NextCommandMailbox; - CommandMailbox->Common.CommandIdentifier = Command->CommandIdentifier; - DAC960_LA_WriteCommandMailbox(NextCommandMailbox, CommandMailbox); - if (Controller->V1.PreviousCommandMailbox1->Words[0] == 0 || - Controller->V1.PreviousCommandMailbox2->Words[0] == 0) - DAC960_LA_MemoryMailboxNewCommand(ControllerBaseAddress); - Controller->V1.PreviousCommandMailbox2 = - Controller->V1.PreviousCommandMailbox1; - Controller->V1.PreviousCommandMailbox1 = NextCommandMailbox; - if (++NextCommandMailbox > Controller->V1.LastCommandMailbox) - NextCommandMailbox = Controller->V1.FirstCommandMailbox; - Controller->V1.NextCommandMailbox = NextCommandMailbox; -} - - -/* - DAC960_LA_QueueCommandSingleMode queues Command for DAC960 LA Series - Controllers with Single Mode Firmware. -*/ - -static void DAC960_LA_QueueCommandSingleMode(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - DAC960_V1_CommandMailbox_T *CommandMailbox = &Command->V1.CommandMailbox; - DAC960_V1_CommandMailbox_T *NextCommandMailbox = - Controller->V1.NextCommandMailbox; - CommandMailbox->Common.CommandIdentifier = Command->CommandIdentifier; - DAC960_LA_WriteCommandMailbox(NextCommandMailbox, CommandMailbox); - if (Controller->V1.PreviousCommandMailbox1->Words[0] == 0 || - Controller->V1.PreviousCommandMailbox2->Words[0] == 0) - DAC960_LA_HardwareMailboxNewCommand(ControllerBaseAddress); - Controller->V1.PreviousCommandMailbox2 = - Controller->V1.PreviousCommandMailbox1; - Controller->V1.PreviousCommandMailbox1 = NextCommandMailbox; - if (++NextCommandMailbox > Controller->V1.LastCommandMailbox) - NextCommandMailbox = Controller->V1.FirstCommandMailbox; - Controller->V1.NextCommandMailbox = NextCommandMailbox; -} - - -/* - DAC960_PG_QueueCommandDualMode queues Command for DAC960 PG Series - Controllers with Dual Mode Firmware. -*/ - -static void DAC960_PG_QueueCommandDualMode(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - DAC960_V1_CommandMailbox_T *CommandMailbox = &Command->V1.CommandMailbox; - DAC960_V1_CommandMailbox_T *NextCommandMailbox = - Controller->V1.NextCommandMailbox; - CommandMailbox->Common.CommandIdentifier = Command->CommandIdentifier; - DAC960_PG_WriteCommandMailbox(NextCommandMailbox, CommandMailbox); - if (Controller->V1.PreviousCommandMailbox1->Words[0] == 0 || - Controller->V1.PreviousCommandMailbox2->Words[0] == 0) - DAC960_PG_MemoryMailboxNewCommand(ControllerBaseAddress); - Controller->V1.PreviousCommandMailbox2 = - Controller->V1.PreviousCommandMailbox1; - Controller->V1.PreviousCommandMailbox1 = NextCommandMailbox; - if (++NextCommandMailbox > Controller->V1.LastCommandMailbox) - NextCommandMailbox = Controller->V1.FirstCommandMailbox; - Controller->V1.NextCommandMailbox = NextCommandMailbox; -} - - -/* - DAC960_PG_QueueCommandSingleMode queues Command for DAC960 PG Series - Controllers with Single Mode Firmware. -*/ - -static void DAC960_PG_QueueCommandSingleMode(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - DAC960_V1_CommandMailbox_T *CommandMailbox = &Command->V1.CommandMailbox; - DAC960_V1_CommandMailbox_T *NextCommandMailbox = - Controller->V1.NextCommandMailbox; - CommandMailbox->Common.CommandIdentifier = Command->CommandIdentifier; - DAC960_PG_WriteCommandMailbox(NextCommandMailbox, CommandMailbox); - if (Controller->V1.PreviousCommandMailbox1->Words[0] == 0 || - Controller->V1.PreviousCommandMailbox2->Words[0] == 0) - DAC960_PG_HardwareMailboxNewCommand(ControllerBaseAddress); - Controller->V1.PreviousCommandMailbox2 = - Controller->V1.PreviousCommandMailbox1; - Controller->V1.PreviousCommandMailbox1 = NextCommandMailbox; - if (++NextCommandMailbox > Controller->V1.LastCommandMailbox) - NextCommandMailbox = Controller->V1.FirstCommandMailbox; - Controller->V1.NextCommandMailbox = NextCommandMailbox; -} - - -/* - DAC960_PD_QueueCommand queues Command for DAC960 PD Series Controllers. -*/ - -static void DAC960_PD_QueueCommand(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - DAC960_V1_CommandMailbox_T *CommandMailbox = &Command->V1.CommandMailbox; - CommandMailbox->Common.CommandIdentifier = Command->CommandIdentifier; - while (DAC960_PD_MailboxFullP(ControllerBaseAddress)) - udelay(1); - DAC960_PD_WriteCommandMailbox(ControllerBaseAddress, CommandMailbox); - DAC960_PD_NewCommand(ControllerBaseAddress); -} - - -/* - DAC960_P_QueueCommand queues Command for DAC960 P Series Controllers. -*/ - -static void DAC960_P_QueueCommand(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - DAC960_V1_CommandMailbox_T *CommandMailbox = &Command->V1.CommandMailbox; - CommandMailbox->Common.CommandIdentifier = Command->CommandIdentifier; - switch (CommandMailbox->Common.CommandOpcode) - { - case DAC960_V1_Enquiry: - CommandMailbox->Common.CommandOpcode = DAC960_V1_Enquiry_Old; - break; - case DAC960_V1_GetDeviceState: - CommandMailbox->Common.CommandOpcode = DAC960_V1_GetDeviceState_Old; - break; - case DAC960_V1_Read: - CommandMailbox->Common.CommandOpcode = DAC960_V1_Read_Old; - DAC960_PD_To_P_TranslateReadWriteCommand(CommandMailbox); - break; - case DAC960_V1_Write: - CommandMailbox->Common.CommandOpcode = DAC960_V1_Write_Old; - DAC960_PD_To_P_TranslateReadWriteCommand(CommandMailbox); - break; - case DAC960_V1_ReadWithScatterGather: - CommandMailbox->Common.CommandOpcode = - DAC960_V1_ReadWithScatterGather_Old; - DAC960_PD_To_P_TranslateReadWriteCommand(CommandMailbox); - break; - case DAC960_V1_WriteWithScatterGather: - CommandMailbox->Common.CommandOpcode = - DAC960_V1_WriteWithScatterGather_Old; - DAC960_PD_To_P_TranslateReadWriteCommand(CommandMailbox); - break; - default: - break; - } - while (DAC960_PD_MailboxFullP(ControllerBaseAddress)) - udelay(1); - DAC960_PD_WriteCommandMailbox(ControllerBaseAddress, CommandMailbox); - DAC960_PD_NewCommand(ControllerBaseAddress); -} - - -/* - DAC960_ExecuteCommand executes Command and waits for completion. -*/ - -static void DAC960_ExecuteCommand(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - DECLARE_COMPLETION_ONSTACK(Completion); - unsigned long flags; - Command->Completion = &Completion; - - spin_lock_irqsave(&Controller->queue_lock, flags); - DAC960_QueueCommand(Command); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - - if (in_interrupt()) - return; - wait_for_completion(&Completion); -} - - -/* - DAC960_V1_ExecuteType3 executes a DAC960 V1 Firmware Controller Type 3 - Command and waits for completion. It returns true on success and false - on failure. -*/ - -static bool DAC960_V1_ExecuteType3(DAC960_Controller_T *Controller, - DAC960_V1_CommandOpcode_T CommandOpcode, - dma_addr_t DataDMA) -{ - DAC960_Command_T *Command = DAC960_AllocateCommand(Controller); - DAC960_V1_CommandMailbox_T *CommandMailbox = &Command->V1.CommandMailbox; - DAC960_V1_CommandStatus_T CommandStatus; - DAC960_V1_ClearCommand(Command); - Command->CommandType = DAC960_ImmediateCommand; - CommandMailbox->Type3.CommandOpcode = CommandOpcode; - CommandMailbox->Type3.BusAddress = DataDMA; - DAC960_ExecuteCommand(Command); - CommandStatus = Command->V1.CommandStatus; - DAC960_DeallocateCommand(Command); - return (CommandStatus == DAC960_V1_NormalCompletion); -} - - -/* - DAC960_V1_ExecuteTypeB executes a DAC960 V1 Firmware Controller Type 3B - Command and waits for completion. It returns true on success and false - on failure. -*/ - -static bool DAC960_V1_ExecuteType3B(DAC960_Controller_T *Controller, - DAC960_V1_CommandOpcode_T CommandOpcode, - unsigned char CommandOpcode2, - dma_addr_t DataDMA) -{ - DAC960_Command_T *Command = DAC960_AllocateCommand(Controller); - DAC960_V1_CommandMailbox_T *CommandMailbox = &Command->V1.CommandMailbox; - DAC960_V1_CommandStatus_T CommandStatus; - DAC960_V1_ClearCommand(Command); - Command->CommandType = DAC960_ImmediateCommand; - CommandMailbox->Type3B.CommandOpcode = CommandOpcode; - CommandMailbox->Type3B.CommandOpcode2 = CommandOpcode2; - CommandMailbox->Type3B.BusAddress = DataDMA; - DAC960_ExecuteCommand(Command); - CommandStatus = Command->V1.CommandStatus; - DAC960_DeallocateCommand(Command); - return (CommandStatus == DAC960_V1_NormalCompletion); -} - - -/* - DAC960_V1_ExecuteType3D executes a DAC960 V1 Firmware Controller Type 3D - Command and waits for completion. It returns true on success and false - on failure. -*/ - -static bool DAC960_V1_ExecuteType3D(DAC960_Controller_T *Controller, - DAC960_V1_CommandOpcode_T CommandOpcode, - unsigned char Channel, - unsigned char TargetID, - dma_addr_t DataDMA) -{ - DAC960_Command_T *Command = DAC960_AllocateCommand(Controller); - DAC960_V1_CommandMailbox_T *CommandMailbox = &Command->V1.CommandMailbox; - DAC960_V1_CommandStatus_T CommandStatus; - DAC960_V1_ClearCommand(Command); - Command->CommandType = DAC960_ImmediateCommand; - CommandMailbox->Type3D.CommandOpcode = CommandOpcode; - CommandMailbox->Type3D.Channel = Channel; - CommandMailbox->Type3D.TargetID = TargetID; - CommandMailbox->Type3D.BusAddress = DataDMA; - DAC960_ExecuteCommand(Command); - CommandStatus = Command->V1.CommandStatus; - DAC960_DeallocateCommand(Command); - return (CommandStatus == DAC960_V1_NormalCompletion); -} - - -/* - DAC960_V2_GeneralInfo executes a DAC960 V2 Firmware General Information - Reading IOCTL Command and waits for completion. It returns true on success - and false on failure. - - Return data in The controller's HealthStatusBuffer, which is dma-able memory -*/ - -static bool DAC960_V2_GeneralInfo(DAC960_Controller_T *Controller) -{ - DAC960_Command_T *Command = DAC960_AllocateCommand(Controller); - DAC960_V2_CommandMailbox_T *CommandMailbox = &Command->V2.CommandMailbox; - DAC960_V2_CommandStatus_T CommandStatus; - DAC960_V2_ClearCommand(Command); - Command->CommandType = DAC960_ImmediateCommand; - CommandMailbox->Common.CommandOpcode = DAC960_V2_IOCTL; - CommandMailbox->Common.CommandControlBits - .DataTransferControllerToHost = true; - CommandMailbox->Common.CommandControlBits - .NoAutoRequestSense = true; - CommandMailbox->Common.DataTransferSize = sizeof(DAC960_V2_HealthStatusBuffer_T); - CommandMailbox->Common.IOCTL_Opcode = DAC960_V2_GetHealthStatus; - CommandMailbox->Common.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentDataPointer = - Controller->V2.HealthStatusBufferDMA; - CommandMailbox->Common.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentByteCount = - CommandMailbox->Common.DataTransferSize; - DAC960_ExecuteCommand(Command); - CommandStatus = Command->V2.CommandStatus; - DAC960_DeallocateCommand(Command); - return (CommandStatus == DAC960_V2_NormalCompletion); -} - - -/* - DAC960_V2_ControllerInfo executes a DAC960 V2 Firmware Controller - Information Reading IOCTL Command and waits for completion. It returns - true on success and false on failure. - - Data is returned in the controller's V2.NewControllerInformation dma-able - memory buffer. -*/ - -static bool DAC960_V2_NewControllerInfo(DAC960_Controller_T *Controller) -{ - DAC960_Command_T *Command = DAC960_AllocateCommand(Controller); - DAC960_V2_CommandMailbox_T *CommandMailbox = &Command->V2.CommandMailbox; - DAC960_V2_CommandStatus_T CommandStatus; - DAC960_V2_ClearCommand(Command); - Command->CommandType = DAC960_ImmediateCommand; - CommandMailbox->ControllerInfo.CommandOpcode = DAC960_V2_IOCTL; - CommandMailbox->ControllerInfo.CommandControlBits - .DataTransferControllerToHost = true; - CommandMailbox->ControllerInfo.CommandControlBits - .NoAutoRequestSense = true; - CommandMailbox->ControllerInfo.DataTransferSize = sizeof(DAC960_V2_ControllerInfo_T); - CommandMailbox->ControllerInfo.ControllerNumber = 0; - CommandMailbox->ControllerInfo.IOCTL_Opcode = DAC960_V2_GetControllerInfo; - CommandMailbox->ControllerInfo.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentDataPointer = - Controller->V2.NewControllerInformationDMA; - CommandMailbox->ControllerInfo.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentByteCount = - CommandMailbox->ControllerInfo.DataTransferSize; - DAC960_ExecuteCommand(Command); - CommandStatus = Command->V2.CommandStatus; - DAC960_DeallocateCommand(Command); - return (CommandStatus == DAC960_V2_NormalCompletion); -} - - -/* - DAC960_V2_LogicalDeviceInfo executes a DAC960 V2 Firmware Controller Logical - Device Information Reading IOCTL Command and waits for completion. It - returns true on success and false on failure. - - Data is returned in the controller's V2.NewLogicalDeviceInformation -*/ - -static bool DAC960_V2_NewLogicalDeviceInfo(DAC960_Controller_T *Controller, - unsigned short LogicalDeviceNumber) -{ - DAC960_Command_T *Command = DAC960_AllocateCommand(Controller); - DAC960_V2_CommandMailbox_T *CommandMailbox = &Command->V2.CommandMailbox; - DAC960_V2_CommandStatus_T CommandStatus; - - DAC960_V2_ClearCommand(Command); - Command->CommandType = DAC960_ImmediateCommand; - CommandMailbox->LogicalDeviceInfo.CommandOpcode = - DAC960_V2_IOCTL; - CommandMailbox->LogicalDeviceInfo.CommandControlBits - .DataTransferControllerToHost = true; - CommandMailbox->LogicalDeviceInfo.CommandControlBits - .NoAutoRequestSense = true; - CommandMailbox->LogicalDeviceInfo.DataTransferSize = - sizeof(DAC960_V2_LogicalDeviceInfo_T); - CommandMailbox->LogicalDeviceInfo.LogicalDevice.LogicalDeviceNumber = - LogicalDeviceNumber; - CommandMailbox->LogicalDeviceInfo.IOCTL_Opcode = DAC960_V2_GetLogicalDeviceInfoValid; - CommandMailbox->LogicalDeviceInfo.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentDataPointer = - Controller->V2.NewLogicalDeviceInformationDMA; - CommandMailbox->LogicalDeviceInfo.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentByteCount = - CommandMailbox->LogicalDeviceInfo.DataTransferSize; - DAC960_ExecuteCommand(Command); - CommandStatus = Command->V2.CommandStatus; - DAC960_DeallocateCommand(Command); - return (CommandStatus == DAC960_V2_NormalCompletion); -} - - -/* - DAC960_V2_PhysicalDeviceInfo executes a DAC960 V2 Firmware Controller "Read - Physical Device Information" IOCTL Command and waits for completion. It - returns true on success and false on failure. - - The Channel, TargetID, LogicalUnit arguments should be 0 the first time - this function is called for a given controller. This will return data - for the "first" device on that controller. The returned data includes a - Channel, TargetID, LogicalUnit that can be passed in to this routine to - get data for the NEXT device on that controller. - - Data is stored in the controller's V2.NewPhysicalDeviceInfo dma-able - memory buffer. - -*/ - -static bool DAC960_V2_NewPhysicalDeviceInfo(DAC960_Controller_T *Controller, - unsigned char Channel, - unsigned char TargetID, - unsigned char LogicalUnit) -{ - DAC960_Command_T *Command = DAC960_AllocateCommand(Controller); - DAC960_V2_CommandMailbox_T *CommandMailbox = &Command->V2.CommandMailbox; - DAC960_V2_CommandStatus_T CommandStatus; - - DAC960_V2_ClearCommand(Command); - Command->CommandType = DAC960_ImmediateCommand; - CommandMailbox->PhysicalDeviceInfo.CommandOpcode = DAC960_V2_IOCTL; - CommandMailbox->PhysicalDeviceInfo.CommandControlBits - .DataTransferControllerToHost = true; - CommandMailbox->PhysicalDeviceInfo.CommandControlBits - .NoAutoRequestSense = true; - CommandMailbox->PhysicalDeviceInfo.DataTransferSize = - sizeof(DAC960_V2_PhysicalDeviceInfo_T); - CommandMailbox->PhysicalDeviceInfo.PhysicalDevice.LogicalUnit = LogicalUnit; - CommandMailbox->PhysicalDeviceInfo.PhysicalDevice.TargetID = TargetID; - CommandMailbox->PhysicalDeviceInfo.PhysicalDevice.Channel = Channel; - CommandMailbox->PhysicalDeviceInfo.IOCTL_Opcode = - DAC960_V2_GetPhysicalDeviceInfoValid; - CommandMailbox->PhysicalDeviceInfo.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentDataPointer = - Controller->V2.NewPhysicalDeviceInformationDMA; - CommandMailbox->PhysicalDeviceInfo.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentByteCount = - CommandMailbox->PhysicalDeviceInfo.DataTransferSize; - DAC960_ExecuteCommand(Command); - CommandStatus = Command->V2.CommandStatus; - DAC960_DeallocateCommand(Command); - return (CommandStatus == DAC960_V2_NormalCompletion); -} - - -static void DAC960_V2_ConstructNewUnitSerialNumber( - DAC960_Controller_T *Controller, - DAC960_V2_CommandMailbox_T *CommandMailbox, int Channel, int TargetID, - int LogicalUnit) -{ - CommandMailbox->SCSI_10.CommandOpcode = DAC960_V2_SCSI_10_Passthru; - CommandMailbox->SCSI_10.CommandControlBits - .DataTransferControllerToHost = true; - CommandMailbox->SCSI_10.CommandControlBits - .NoAutoRequestSense = true; - CommandMailbox->SCSI_10.DataTransferSize = - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T); - CommandMailbox->SCSI_10.PhysicalDevice.LogicalUnit = LogicalUnit; - CommandMailbox->SCSI_10.PhysicalDevice.TargetID = TargetID; - CommandMailbox->SCSI_10.PhysicalDevice.Channel = Channel; - CommandMailbox->SCSI_10.CDBLength = 6; - CommandMailbox->SCSI_10.SCSI_CDB[0] = 0x12; /* INQUIRY */ - CommandMailbox->SCSI_10.SCSI_CDB[1] = 1; /* EVPD = 1 */ - CommandMailbox->SCSI_10.SCSI_CDB[2] = 0x80; /* Page Code */ - CommandMailbox->SCSI_10.SCSI_CDB[3] = 0; /* Reserved */ - CommandMailbox->SCSI_10.SCSI_CDB[4] = - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T); - CommandMailbox->SCSI_10.SCSI_CDB[5] = 0; /* Control */ - CommandMailbox->SCSI_10.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentDataPointer = - Controller->V2.NewInquiryUnitSerialNumberDMA; - CommandMailbox->SCSI_10.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentByteCount = - CommandMailbox->SCSI_10.DataTransferSize; -} - - -/* - DAC960_V2_NewUnitSerialNumber executes an SCSI pass-through - Inquiry command to a SCSI device identified by Channel number, - Target id, Logical Unit Number. This function Waits for completion - of the command. - - The return data includes Unit Serial Number information for the - specified device. - - Data is stored in the controller's V2.NewPhysicalDeviceInfo dma-able - memory buffer. -*/ - -static bool DAC960_V2_NewInquiryUnitSerialNumber(DAC960_Controller_T *Controller, - int Channel, int TargetID, int LogicalUnit) -{ - DAC960_Command_T *Command; - DAC960_V2_CommandMailbox_T *CommandMailbox; - DAC960_V2_CommandStatus_T CommandStatus; - - Command = DAC960_AllocateCommand(Controller); - CommandMailbox = &Command->V2.CommandMailbox; - DAC960_V2_ClearCommand(Command); - Command->CommandType = DAC960_ImmediateCommand; - - DAC960_V2_ConstructNewUnitSerialNumber(Controller, CommandMailbox, - Channel, TargetID, LogicalUnit); - - DAC960_ExecuteCommand(Command); - CommandStatus = Command->V2.CommandStatus; - DAC960_DeallocateCommand(Command); - return (CommandStatus == DAC960_V2_NormalCompletion); -} - - -/* - DAC960_V2_DeviceOperation executes a DAC960 V2 Firmware Controller Device - Operation IOCTL Command and waits for completion. It returns true on - success and false on failure. -*/ - -static bool DAC960_V2_DeviceOperation(DAC960_Controller_T *Controller, - DAC960_V2_IOCTL_Opcode_T IOCTL_Opcode, - DAC960_V2_OperationDevice_T - OperationDevice) -{ - DAC960_Command_T *Command = DAC960_AllocateCommand(Controller); - DAC960_V2_CommandMailbox_T *CommandMailbox = &Command->V2.CommandMailbox; - DAC960_V2_CommandStatus_T CommandStatus; - DAC960_V2_ClearCommand(Command); - Command->CommandType = DAC960_ImmediateCommand; - CommandMailbox->DeviceOperation.CommandOpcode = DAC960_V2_IOCTL; - CommandMailbox->DeviceOperation.CommandControlBits - .DataTransferControllerToHost = true; - CommandMailbox->DeviceOperation.CommandControlBits - .NoAutoRequestSense = true; - CommandMailbox->DeviceOperation.IOCTL_Opcode = IOCTL_Opcode; - CommandMailbox->DeviceOperation.OperationDevice = OperationDevice; - DAC960_ExecuteCommand(Command); - CommandStatus = Command->V2.CommandStatus; - DAC960_DeallocateCommand(Command); - return (CommandStatus == DAC960_V2_NormalCompletion); -} - - -/* - DAC960_V1_EnableMemoryMailboxInterface enables the Memory Mailbox Interface - for DAC960 V1 Firmware Controllers. - - PD and P controller types have no memory mailbox, but still need the - other dma mapped memory. -*/ - -static bool DAC960_V1_EnableMemoryMailboxInterface(DAC960_Controller_T - *Controller) -{ - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - DAC960_HardwareType_T hw_type = Controller->HardwareType; - struct pci_dev *PCI_Device = Controller->PCIDevice; - struct dma_loaf *DmaPages = &Controller->DmaPages; - size_t DmaPagesSize; - size_t CommandMailboxesSize; - size_t StatusMailboxesSize; - - DAC960_V1_CommandMailbox_T *CommandMailboxesMemory; - dma_addr_t CommandMailboxesMemoryDMA; - - DAC960_V1_StatusMailbox_T *StatusMailboxesMemory; - dma_addr_t StatusMailboxesMemoryDMA; - - DAC960_V1_CommandMailbox_T CommandMailbox; - DAC960_V1_CommandStatus_T CommandStatus; - int TimeoutCounter; - int i; - - memset(&CommandMailbox, 0, sizeof(DAC960_V1_CommandMailbox_T)); - - if (pci_set_dma_mask(Controller->PCIDevice, DMA_BIT_MASK(32))) - return DAC960_Failure(Controller, "DMA mask out of range"); - - if ((hw_type == DAC960_PD_Controller) || (hw_type == DAC960_P_Controller)) { - CommandMailboxesSize = 0; - StatusMailboxesSize = 0; - } else { - CommandMailboxesSize = DAC960_V1_CommandMailboxCount * sizeof(DAC960_V1_CommandMailbox_T); - StatusMailboxesSize = DAC960_V1_StatusMailboxCount * sizeof(DAC960_V1_StatusMailbox_T); - } - DmaPagesSize = CommandMailboxesSize + StatusMailboxesSize + - sizeof(DAC960_V1_DCDB_T) + sizeof(DAC960_V1_Enquiry_T) + - sizeof(DAC960_V1_ErrorTable_T) + sizeof(DAC960_V1_EventLogEntry_T) + - sizeof(DAC960_V1_RebuildProgress_T) + - sizeof(DAC960_V1_LogicalDriveInformationArray_T) + - sizeof(DAC960_V1_BackgroundInitializationStatus_T) + - sizeof(DAC960_V1_DeviceState_T) + sizeof(DAC960_SCSI_Inquiry_T) + - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T); - - if (!init_dma_loaf(PCI_Device, DmaPages, DmaPagesSize)) - return false; - - - if ((hw_type == DAC960_PD_Controller) || (hw_type == DAC960_P_Controller)) - goto skip_mailboxes; - - CommandMailboxesMemory = slice_dma_loaf(DmaPages, - CommandMailboxesSize, &CommandMailboxesMemoryDMA); - - /* These are the base addresses for the command memory mailbox array */ - Controller->V1.FirstCommandMailbox = CommandMailboxesMemory; - Controller->V1.FirstCommandMailboxDMA = CommandMailboxesMemoryDMA; - - CommandMailboxesMemory += DAC960_V1_CommandMailboxCount - 1; - Controller->V1.LastCommandMailbox = CommandMailboxesMemory; - Controller->V1.NextCommandMailbox = Controller->V1.FirstCommandMailbox; - Controller->V1.PreviousCommandMailbox1 = Controller->V1.LastCommandMailbox; - Controller->V1.PreviousCommandMailbox2 = - Controller->V1.LastCommandMailbox - 1; - - /* These are the base addresses for the status memory mailbox array */ - StatusMailboxesMemory = slice_dma_loaf(DmaPages, - StatusMailboxesSize, &StatusMailboxesMemoryDMA); - - Controller->V1.FirstStatusMailbox = StatusMailboxesMemory; - Controller->V1.FirstStatusMailboxDMA = StatusMailboxesMemoryDMA; - StatusMailboxesMemory += DAC960_V1_StatusMailboxCount - 1; - Controller->V1.LastStatusMailbox = StatusMailboxesMemory; - Controller->V1.NextStatusMailbox = Controller->V1.FirstStatusMailbox; - -skip_mailboxes: - Controller->V1.MonitoringDCDB = slice_dma_loaf(DmaPages, - sizeof(DAC960_V1_DCDB_T), - &Controller->V1.MonitoringDCDB_DMA); - - Controller->V1.NewEnquiry = slice_dma_loaf(DmaPages, - sizeof(DAC960_V1_Enquiry_T), - &Controller->V1.NewEnquiryDMA); - - Controller->V1.NewErrorTable = slice_dma_loaf(DmaPages, - sizeof(DAC960_V1_ErrorTable_T), - &Controller->V1.NewErrorTableDMA); - - Controller->V1.EventLogEntry = slice_dma_loaf(DmaPages, - sizeof(DAC960_V1_EventLogEntry_T), - &Controller->V1.EventLogEntryDMA); - - Controller->V1.RebuildProgress = slice_dma_loaf(DmaPages, - sizeof(DAC960_V1_RebuildProgress_T), - &Controller->V1.RebuildProgressDMA); - - Controller->V1.NewLogicalDriveInformation = slice_dma_loaf(DmaPages, - sizeof(DAC960_V1_LogicalDriveInformationArray_T), - &Controller->V1.NewLogicalDriveInformationDMA); - - Controller->V1.BackgroundInitializationStatus = slice_dma_loaf(DmaPages, - sizeof(DAC960_V1_BackgroundInitializationStatus_T), - &Controller->V1.BackgroundInitializationStatusDMA); - - Controller->V1.NewDeviceState = slice_dma_loaf(DmaPages, - sizeof(DAC960_V1_DeviceState_T), - &Controller->V1.NewDeviceStateDMA); - - Controller->V1.NewInquiryStandardData = slice_dma_loaf(DmaPages, - sizeof(DAC960_SCSI_Inquiry_T), - &Controller->V1.NewInquiryStandardDataDMA); - - Controller->V1.NewInquiryUnitSerialNumber = slice_dma_loaf(DmaPages, - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T), - &Controller->V1.NewInquiryUnitSerialNumberDMA); - - if ((hw_type == DAC960_PD_Controller) || (hw_type == DAC960_P_Controller)) - return true; - - /* Enable the Memory Mailbox Interface. */ - Controller->V1.DualModeMemoryMailboxInterface = true; - CommandMailbox.TypeX.CommandOpcode = 0x2B; - CommandMailbox.TypeX.CommandIdentifier = 0; - CommandMailbox.TypeX.CommandOpcode2 = 0x14; - CommandMailbox.TypeX.CommandMailboxesBusAddress = - Controller->V1.FirstCommandMailboxDMA; - CommandMailbox.TypeX.StatusMailboxesBusAddress = - Controller->V1.FirstStatusMailboxDMA; -#define TIMEOUT_COUNT 1000000 - - for (i = 0; i < 2; i++) - switch (Controller->HardwareType) - { - case DAC960_LA_Controller: - TimeoutCounter = TIMEOUT_COUNT; - while (--TimeoutCounter >= 0) - { - if (!DAC960_LA_HardwareMailboxFullP(ControllerBaseAddress)) - break; - udelay(10); - } - if (TimeoutCounter < 0) return false; - DAC960_LA_WriteHardwareMailbox(ControllerBaseAddress, &CommandMailbox); - DAC960_LA_HardwareMailboxNewCommand(ControllerBaseAddress); - TimeoutCounter = TIMEOUT_COUNT; - while (--TimeoutCounter >= 0) - { - if (DAC960_LA_HardwareMailboxStatusAvailableP( - ControllerBaseAddress)) - break; - udelay(10); - } - if (TimeoutCounter < 0) return false; - CommandStatus = DAC960_LA_ReadStatusRegister(ControllerBaseAddress); - DAC960_LA_AcknowledgeHardwareMailboxInterrupt(ControllerBaseAddress); - DAC960_LA_AcknowledgeHardwareMailboxStatus(ControllerBaseAddress); - if (CommandStatus == DAC960_V1_NormalCompletion) return true; - Controller->V1.DualModeMemoryMailboxInterface = false; - CommandMailbox.TypeX.CommandOpcode2 = 0x10; - break; - case DAC960_PG_Controller: - TimeoutCounter = TIMEOUT_COUNT; - while (--TimeoutCounter >= 0) - { - if (!DAC960_PG_HardwareMailboxFullP(ControllerBaseAddress)) - break; - udelay(10); - } - if (TimeoutCounter < 0) return false; - DAC960_PG_WriteHardwareMailbox(ControllerBaseAddress, &CommandMailbox); - DAC960_PG_HardwareMailboxNewCommand(ControllerBaseAddress); - - TimeoutCounter = TIMEOUT_COUNT; - while (--TimeoutCounter >= 0) - { - if (DAC960_PG_HardwareMailboxStatusAvailableP( - ControllerBaseAddress)) - break; - udelay(10); - } - if (TimeoutCounter < 0) return false; - CommandStatus = DAC960_PG_ReadStatusRegister(ControllerBaseAddress); - DAC960_PG_AcknowledgeHardwareMailboxInterrupt(ControllerBaseAddress); - DAC960_PG_AcknowledgeHardwareMailboxStatus(ControllerBaseAddress); - if (CommandStatus == DAC960_V1_NormalCompletion) return true; - Controller->V1.DualModeMemoryMailboxInterface = false; - CommandMailbox.TypeX.CommandOpcode2 = 0x10; - break; - default: - DAC960_Failure(Controller, "Unknown Controller Type\n"); - break; - } - return false; -} - - -/* - DAC960_V2_EnableMemoryMailboxInterface enables the Memory Mailbox Interface - for DAC960 V2 Firmware Controllers. - - Aggregate the space needed for the controller's memory mailbox and - the other data structures that will be targets of dma transfers with - the controller. Allocate a dma-mapped region of memory to hold these - structures. Then, save CPU pointers and dma_addr_t values to reference - the structures that are contained in that region. -*/ - -static bool DAC960_V2_EnableMemoryMailboxInterface(DAC960_Controller_T - *Controller) -{ - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - struct pci_dev *PCI_Device = Controller->PCIDevice; - struct dma_loaf *DmaPages = &Controller->DmaPages; - size_t DmaPagesSize; - size_t CommandMailboxesSize; - size_t StatusMailboxesSize; - - DAC960_V2_CommandMailbox_T *CommandMailboxesMemory; - dma_addr_t CommandMailboxesMemoryDMA; - - DAC960_V2_StatusMailbox_T *StatusMailboxesMemory; - dma_addr_t StatusMailboxesMemoryDMA; - - DAC960_V2_CommandMailbox_T *CommandMailbox; - dma_addr_t CommandMailboxDMA; - DAC960_V2_CommandStatus_T CommandStatus; - - if (pci_set_dma_mask(Controller->PCIDevice, DMA_BIT_MASK(64)) && - pci_set_dma_mask(Controller->PCIDevice, DMA_BIT_MASK(32))) - return DAC960_Failure(Controller, "DMA mask out of range"); - - /* This is a temporary dma mapping, used only in the scope of this function */ - CommandMailbox = pci_alloc_consistent(PCI_Device, - sizeof(DAC960_V2_CommandMailbox_T), &CommandMailboxDMA); - if (CommandMailbox == NULL) - return false; - - CommandMailboxesSize = DAC960_V2_CommandMailboxCount * sizeof(DAC960_V2_CommandMailbox_T); - StatusMailboxesSize = DAC960_V2_StatusMailboxCount * sizeof(DAC960_V2_StatusMailbox_T); - DmaPagesSize = - CommandMailboxesSize + StatusMailboxesSize + - sizeof(DAC960_V2_HealthStatusBuffer_T) + - sizeof(DAC960_V2_ControllerInfo_T) + - sizeof(DAC960_V2_LogicalDeviceInfo_T) + - sizeof(DAC960_V2_PhysicalDeviceInfo_T) + - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T) + - sizeof(DAC960_V2_Event_T) + - sizeof(DAC960_V2_PhysicalToLogicalDevice_T); - - if (!init_dma_loaf(PCI_Device, DmaPages, DmaPagesSize)) { - pci_free_consistent(PCI_Device, sizeof(DAC960_V2_CommandMailbox_T), - CommandMailbox, CommandMailboxDMA); - return false; - } - - CommandMailboxesMemory = slice_dma_loaf(DmaPages, - CommandMailboxesSize, &CommandMailboxesMemoryDMA); - - /* These are the base addresses for the command memory mailbox array */ - Controller->V2.FirstCommandMailbox = CommandMailboxesMemory; - Controller->V2.FirstCommandMailboxDMA = CommandMailboxesMemoryDMA; - - CommandMailboxesMemory += DAC960_V2_CommandMailboxCount - 1; - Controller->V2.LastCommandMailbox = CommandMailboxesMemory; - Controller->V2.NextCommandMailbox = Controller->V2.FirstCommandMailbox; - Controller->V2.PreviousCommandMailbox1 = Controller->V2.LastCommandMailbox; - Controller->V2.PreviousCommandMailbox2 = - Controller->V2.LastCommandMailbox - 1; - - /* These are the base addresses for the status memory mailbox array */ - StatusMailboxesMemory = slice_dma_loaf(DmaPages, - StatusMailboxesSize, &StatusMailboxesMemoryDMA); - - Controller->V2.FirstStatusMailbox = StatusMailboxesMemory; - Controller->V2.FirstStatusMailboxDMA = StatusMailboxesMemoryDMA; - StatusMailboxesMemory += DAC960_V2_StatusMailboxCount - 1; - Controller->V2.LastStatusMailbox = StatusMailboxesMemory; - Controller->V2.NextStatusMailbox = Controller->V2.FirstStatusMailbox; - - Controller->V2.HealthStatusBuffer = slice_dma_loaf(DmaPages, - sizeof(DAC960_V2_HealthStatusBuffer_T), - &Controller->V2.HealthStatusBufferDMA); - - Controller->V2.NewControllerInformation = slice_dma_loaf(DmaPages, - sizeof(DAC960_V2_ControllerInfo_T), - &Controller->V2.NewControllerInformationDMA); - - Controller->V2.NewLogicalDeviceInformation = slice_dma_loaf(DmaPages, - sizeof(DAC960_V2_LogicalDeviceInfo_T), - &Controller->V2.NewLogicalDeviceInformationDMA); - - Controller->V2.NewPhysicalDeviceInformation = slice_dma_loaf(DmaPages, - sizeof(DAC960_V2_PhysicalDeviceInfo_T), - &Controller->V2.NewPhysicalDeviceInformationDMA); - - Controller->V2.NewInquiryUnitSerialNumber = slice_dma_loaf(DmaPages, - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T), - &Controller->V2.NewInquiryUnitSerialNumberDMA); - - Controller->V2.Event = slice_dma_loaf(DmaPages, - sizeof(DAC960_V2_Event_T), - &Controller->V2.EventDMA); - - Controller->V2.PhysicalToLogicalDevice = slice_dma_loaf(DmaPages, - sizeof(DAC960_V2_PhysicalToLogicalDevice_T), - &Controller->V2.PhysicalToLogicalDeviceDMA); - - /* - Enable the Memory Mailbox Interface. - - I don't know why we can't just use one of the memory mailboxes - we just allocated to do this, instead of using this temporary one. - Try this change later. - */ - memset(CommandMailbox, 0, sizeof(DAC960_V2_CommandMailbox_T)); - CommandMailbox->SetMemoryMailbox.CommandIdentifier = 1; - CommandMailbox->SetMemoryMailbox.CommandOpcode = DAC960_V2_IOCTL; - CommandMailbox->SetMemoryMailbox.CommandControlBits.NoAutoRequestSense = true; - CommandMailbox->SetMemoryMailbox.FirstCommandMailboxSizeKB = - (DAC960_V2_CommandMailboxCount * sizeof(DAC960_V2_CommandMailbox_T)) >> 10; - CommandMailbox->SetMemoryMailbox.FirstStatusMailboxSizeKB = - (DAC960_V2_StatusMailboxCount * sizeof(DAC960_V2_StatusMailbox_T)) >> 10; - CommandMailbox->SetMemoryMailbox.SecondCommandMailboxSizeKB = 0; - CommandMailbox->SetMemoryMailbox.SecondStatusMailboxSizeKB = 0; - CommandMailbox->SetMemoryMailbox.RequestSenseSize = 0; - CommandMailbox->SetMemoryMailbox.IOCTL_Opcode = DAC960_V2_SetMemoryMailbox; - CommandMailbox->SetMemoryMailbox.HealthStatusBufferSizeKB = 1; - CommandMailbox->SetMemoryMailbox.HealthStatusBufferBusAddress = - Controller->V2.HealthStatusBufferDMA; - CommandMailbox->SetMemoryMailbox.FirstCommandMailboxBusAddress = - Controller->V2.FirstCommandMailboxDMA; - CommandMailbox->SetMemoryMailbox.FirstStatusMailboxBusAddress = - Controller->V2.FirstStatusMailboxDMA; - switch (Controller->HardwareType) - { - case DAC960_GEM_Controller: - while (DAC960_GEM_HardwareMailboxFullP(ControllerBaseAddress)) - udelay(1); - DAC960_GEM_WriteHardwareMailbox(ControllerBaseAddress, CommandMailboxDMA); - DAC960_GEM_HardwareMailboxNewCommand(ControllerBaseAddress); - while (!DAC960_GEM_HardwareMailboxStatusAvailableP(ControllerBaseAddress)) - udelay(1); - CommandStatus = DAC960_GEM_ReadCommandStatus(ControllerBaseAddress); - DAC960_GEM_AcknowledgeHardwareMailboxInterrupt(ControllerBaseAddress); - DAC960_GEM_AcknowledgeHardwareMailboxStatus(ControllerBaseAddress); - break; - case DAC960_BA_Controller: - while (DAC960_BA_HardwareMailboxFullP(ControllerBaseAddress)) - udelay(1); - DAC960_BA_WriteHardwareMailbox(ControllerBaseAddress, CommandMailboxDMA); - DAC960_BA_HardwareMailboxNewCommand(ControllerBaseAddress); - while (!DAC960_BA_HardwareMailboxStatusAvailableP(ControllerBaseAddress)) - udelay(1); - CommandStatus = DAC960_BA_ReadCommandStatus(ControllerBaseAddress); - DAC960_BA_AcknowledgeHardwareMailboxInterrupt(ControllerBaseAddress); - DAC960_BA_AcknowledgeHardwareMailboxStatus(ControllerBaseAddress); - break; - case DAC960_LP_Controller: - while (DAC960_LP_HardwareMailboxFullP(ControllerBaseAddress)) - udelay(1); - DAC960_LP_WriteHardwareMailbox(ControllerBaseAddress, CommandMailboxDMA); - DAC960_LP_HardwareMailboxNewCommand(ControllerBaseAddress); - while (!DAC960_LP_HardwareMailboxStatusAvailableP(ControllerBaseAddress)) - udelay(1); - CommandStatus = DAC960_LP_ReadCommandStatus(ControllerBaseAddress); - DAC960_LP_AcknowledgeHardwareMailboxInterrupt(ControllerBaseAddress); - DAC960_LP_AcknowledgeHardwareMailboxStatus(ControllerBaseAddress); - break; - default: - DAC960_Failure(Controller, "Unknown Controller Type\n"); - CommandStatus = DAC960_V2_AbormalCompletion; - break; - } - pci_free_consistent(PCI_Device, sizeof(DAC960_V2_CommandMailbox_T), - CommandMailbox, CommandMailboxDMA); - return (CommandStatus == DAC960_V2_NormalCompletion); -} - - -/* - DAC960_V1_ReadControllerConfiguration reads the Configuration Information - from DAC960 V1 Firmware Controllers and initializes the Controller structure. -*/ - -static bool DAC960_V1_ReadControllerConfiguration(DAC960_Controller_T - *Controller) -{ - DAC960_V1_Enquiry2_T *Enquiry2; - dma_addr_t Enquiry2DMA; - DAC960_V1_Config2_T *Config2; - dma_addr_t Config2DMA; - int LogicalDriveNumber, Channel, TargetID; - struct dma_loaf local_dma; - - if (!init_dma_loaf(Controller->PCIDevice, &local_dma, - sizeof(DAC960_V1_Enquiry2_T) + sizeof(DAC960_V1_Config2_T))) - return DAC960_Failure(Controller, "LOGICAL DEVICE ALLOCATION"); - - Enquiry2 = slice_dma_loaf(&local_dma, sizeof(DAC960_V1_Enquiry2_T), &Enquiry2DMA); - Config2 = slice_dma_loaf(&local_dma, sizeof(DAC960_V1_Config2_T), &Config2DMA); - - if (!DAC960_V1_ExecuteType3(Controller, DAC960_V1_Enquiry, - Controller->V1.NewEnquiryDMA)) { - free_dma_loaf(Controller->PCIDevice, &local_dma); - return DAC960_Failure(Controller, "ENQUIRY"); - } - memcpy(&Controller->V1.Enquiry, Controller->V1.NewEnquiry, - sizeof(DAC960_V1_Enquiry_T)); - - if (!DAC960_V1_ExecuteType3(Controller, DAC960_V1_Enquiry2, Enquiry2DMA)) { - free_dma_loaf(Controller->PCIDevice, &local_dma); - return DAC960_Failure(Controller, "ENQUIRY2"); - } - - if (!DAC960_V1_ExecuteType3(Controller, DAC960_V1_ReadConfig2, Config2DMA)) { - free_dma_loaf(Controller->PCIDevice, &local_dma); - return DAC960_Failure(Controller, "READ CONFIG2"); - } - - if (!DAC960_V1_ExecuteType3(Controller, DAC960_V1_GetLogicalDriveInformation, - Controller->V1.NewLogicalDriveInformationDMA)) { - free_dma_loaf(Controller->PCIDevice, &local_dma); - return DAC960_Failure(Controller, "GET LOGICAL DRIVE INFORMATION"); - } - memcpy(&Controller->V1.LogicalDriveInformation, - Controller->V1.NewLogicalDriveInformation, - sizeof(DAC960_V1_LogicalDriveInformationArray_T)); - - for (Channel = 0; Channel < Enquiry2->ActualChannels; Channel++) - for (TargetID = 0; TargetID < Enquiry2->MaxTargets; TargetID++) { - if (!DAC960_V1_ExecuteType3D(Controller, DAC960_V1_GetDeviceState, - Channel, TargetID, - Controller->V1.NewDeviceStateDMA)) { - free_dma_loaf(Controller->PCIDevice, &local_dma); - return DAC960_Failure(Controller, "GET DEVICE STATE"); - } - memcpy(&Controller->V1.DeviceState[Channel][TargetID], - Controller->V1.NewDeviceState, sizeof(DAC960_V1_DeviceState_T)); - } - /* - Initialize the Controller Model Name and Full Model Name fields. - */ - switch (Enquiry2->HardwareID.SubModel) - { - case DAC960_V1_P_PD_PU: - if (Enquiry2->SCSICapability.BusSpeed == DAC960_V1_Ultra) - strcpy(Controller->ModelName, "DAC960PU"); - else strcpy(Controller->ModelName, "DAC960PD"); - break; - case DAC960_V1_PL: - strcpy(Controller->ModelName, "DAC960PL"); - break; - case DAC960_V1_PG: - strcpy(Controller->ModelName, "DAC960PG"); - break; - case DAC960_V1_PJ: - strcpy(Controller->ModelName, "DAC960PJ"); - break; - case DAC960_V1_PR: - strcpy(Controller->ModelName, "DAC960PR"); - break; - case DAC960_V1_PT: - strcpy(Controller->ModelName, "DAC960PT"); - break; - case DAC960_V1_PTL0: - strcpy(Controller->ModelName, "DAC960PTL0"); - break; - case DAC960_V1_PRL: - strcpy(Controller->ModelName, "DAC960PRL"); - break; - case DAC960_V1_PTL1: - strcpy(Controller->ModelName, "DAC960PTL1"); - break; - case DAC960_V1_1164P: - strcpy(Controller->ModelName, "DAC1164P"); - break; - default: - free_dma_loaf(Controller->PCIDevice, &local_dma); - return DAC960_Failure(Controller, "MODEL VERIFICATION"); - } - strcpy(Controller->FullModelName, "Mylex "); - strcat(Controller->FullModelName, Controller->ModelName); - /* - Initialize the Controller Firmware Version field and verify that it - is a supported firmware version. The supported firmware versions are: - - DAC1164P 5.06 and above - DAC960PTL/PRL/PJ/PG 4.06 and above - DAC960PU/PD/PL 3.51 and above - DAC960PU/PD/PL/P 2.73 and above - */ -#if defined(CONFIG_ALPHA) - /* - DEC Alpha machines were often equipped with DAC960 cards that were - OEMed from Mylex, and had their own custom firmware. Version 2.70, - the last custom FW revision to be released by DEC for these older - controllers, appears to work quite well with this driver. - - Cards tested successfully were several versions each of the PD and - PU, called by DEC the KZPSC and KZPAC, respectively, and having - the Manufacturer Numbers (from Mylex), usually on a sticker on the - back of the board, of: - - KZPSC: D040347 (1-channel) or D040348 (2-channel) or D040349 (3-channel) - KZPAC: D040395 (1-channel) or D040396 (2-channel) or D040397 (3-channel) - */ -# define FIRMWARE_27X "2.70" -#else -# define FIRMWARE_27X "2.73" -#endif - - if (Enquiry2->FirmwareID.MajorVersion == 0) - { - Enquiry2->FirmwareID.MajorVersion = - Controller->V1.Enquiry.MajorFirmwareVersion; - Enquiry2->FirmwareID.MinorVersion = - Controller->V1.Enquiry.MinorFirmwareVersion; - Enquiry2->FirmwareID.FirmwareType = '0'; - Enquiry2->FirmwareID.TurnID = 0; - } - snprintf(Controller->FirmwareVersion, sizeof(Controller->FirmwareVersion), - "%d.%02d-%c-%02d", - Enquiry2->FirmwareID.MajorVersion, - Enquiry2->FirmwareID.MinorVersion, - Enquiry2->FirmwareID.FirmwareType, - Enquiry2->FirmwareID.TurnID); - if (!((Controller->FirmwareVersion[0] == '5' && - strcmp(Controller->FirmwareVersion, "5.06") >= 0) || - (Controller->FirmwareVersion[0] == '4' && - strcmp(Controller->FirmwareVersion, "4.06") >= 0) || - (Controller->FirmwareVersion[0] == '3' && - strcmp(Controller->FirmwareVersion, "3.51") >= 0) || - (Controller->FirmwareVersion[0] == '2' && - strcmp(Controller->FirmwareVersion, FIRMWARE_27X) >= 0))) - { - DAC960_Failure(Controller, "FIRMWARE VERSION VERIFICATION"); - DAC960_Error("Firmware Version = '%s'\n", Controller, - Controller->FirmwareVersion); - free_dma_loaf(Controller->PCIDevice, &local_dma); - return false; - } - /* - Initialize the Controller Channels, Targets, Memory Size, and SAF-TE - Enclosure Management Enabled fields. - */ - Controller->Channels = Enquiry2->ActualChannels; - Controller->Targets = Enquiry2->MaxTargets; - Controller->MemorySize = Enquiry2->MemorySize >> 20; - Controller->V1.SAFTE_EnclosureManagementEnabled = - (Enquiry2->FaultManagementType == DAC960_V1_SAFTE); - /* - Initialize the Controller Queue Depth, Driver Queue Depth, Logical Drive - Count, Maximum Blocks per Command, Controller Scatter/Gather Limit, and - Driver Scatter/Gather Limit. The Driver Queue Depth must be at most one - less than the Controller Queue Depth to allow for an automatic drive - rebuild operation. - */ - Controller->ControllerQueueDepth = Controller->V1.Enquiry.MaxCommands; - Controller->DriverQueueDepth = Controller->ControllerQueueDepth - 1; - if (Controller->DriverQueueDepth > DAC960_MaxDriverQueueDepth) - Controller->DriverQueueDepth = DAC960_MaxDriverQueueDepth; - Controller->LogicalDriveCount = - Controller->V1.Enquiry.NumberOfLogicalDrives; - Controller->MaxBlocksPerCommand = Enquiry2->MaxBlocksPerCommand; - Controller->ControllerScatterGatherLimit = Enquiry2->MaxScatterGatherEntries; - Controller->DriverScatterGatherLimit = - Controller->ControllerScatterGatherLimit; - if (Controller->DriverScatterGatherLimit > DAC960_V1_ScatterGatherLimit) - Controller->DriverScatterGatherLimit = DAC960_V1_ScatterGatherLimit; - /* - Initialize the Stripe Size, Segment Size, and Geometry Translation. - */ - Controller->V1.StripeSize = Config2->BlocksPerStripe * Config2->BlockFactor - >> (10 - DAC960_BlockSizeBits); - Controller->V1.SegmentSize = Config2->BlocksPerCacheLine * Config2->BlockFactor - >> (10 - DAC960_BlockSizeBits); - switch (Config2->DriveGeometry) - { - case DAC960_V1_Geometry_128_32: - Controller->V1.GeometryTranslationHeads = 128; - Controller->V1.GeometryTranslationSectors = 32; - break; - case DAC960_V1_Geometry_255_63: - Controller->V1.GeometryTranslationHeads = 255; - Controller->V1.GeometryTranslationSectors = 63; - break; - default: - free_dma_loaf(Controller->PCIDevice, &local_dma); - return DAC960_Failure(Controller, "CONFIG2 DRIVE GEOMETRY"); - } - /* - Initialize the Background Initialization Status. - */ - if ((Controller->FirmwareVersion[0] == '4' && - strcmp(Controller->FirmwareVersion, "4.08") >= 0) || - (Controller->FirmwareVersion[0] == '5' && - strcmp(Controller->FirmwareVersion, "5.08") >= 0)) - { - Controller->V1.BackgroundInitializationStatusSupported = true; - DAC960_V1_ExecuteType3B(Controller, - DAC960_V1_BackgroundInitializationControl, 0x20, - Controller-> - V1.BackgroundInitializationStatusDMA); - memcpy(&Controller->V1.LastBackgroundInitializationStatus, - Controller->V1.BackgroundInitializationStatus, - sizeof(DAC960_V1_BackgroundInitializationStatus_T)); - } - /* - Initialize the Logical Drive Initially Accessible flag. - */ - for (LogicalDriveNumber = 0; - LogicalDriveNumber < Controller->LogicalDriveCount; - LogicalDriveNumber++) - if (Controller->V1.LogicalDriveInformation - [LogicalDriveNumber].LogicalDriveState != - DAC960_V1_LogicalDrive_Offline) - Controller->LogicalDriveInitiallyAccessible[LogicalDriveNumber] = true; - Controller->V1.LastRebuildStatus = DAC960_V1_NoRebuildOrCheckInProgress; - free_dma_loaf(Controller->PCIDevice, &local_dma); - return true; -} - - -/* - DAC960_V2_ReadControllerConfiguration reads the Configuration Information - from DAC960 V2 Firmware Controllers and initializes the Controller structure. -*/ - -static bool DAC960_V2_ReadControllerConfiguration(DAC960_Controller_T - *Controller) -{ - DAC960_V2_ControllerInfo_T *ControllerInfo = - &Controller->V2.ControllerInformation; - unsigned short LogicalDeviceNumber = 0; - int ModelNameLength; - - /* Get data into dma-able area, then copy into permanent location */ - if (!DAC960_V2_NewControllerInfo(Controller)) - return DAC960_Failure(Controller, "GET CONTROLLER INFO"); - memcpy(ControllerInfo, Controller->V2.NewControllerInformation, - sizeof(DAC960_V2_ControllerInfo_T)); - - - if (!DAC960_V2_GeneralInfo(Controller)) - return DAC960_Failure(Controller, "GET HEALTH STATUS"); - - /* - Initialize the Controller Model Name and Full Model Name fields. - */ - ModelNameLength = sizeof(ControllerInfo->ControllerName); - if (ModelNameLength > sizeof(Controller->ModelName)-1) - ModelNameLength = sizeof(Controller->ModelName)-1; - memcpy(Controller->ModelName, ControllerInfo->ControllerName, - ModelNameLength); - ModelNameLength--; - while (Controller->ModelName[ModelNameLength] == ' ' || - Controller->ModelName[ModelNameLength] == '\0') - ModelNameLength--; - Controller->ModelName[++ModelNameLength] = '\0'; - strcpy(Controller->FullModelName, "Mylex "); - strcat(Controller->FullModelName, Controller->ModelName); - /* - Initialize the Controller Firmware Version field. - */ - sprintf(Controller->FirmwareVersion, "%d.%02d-%02d", - ControllerInfo->FirmwareMajorVersion, - ControllerInfo->FirmwareMinorVersion, - ControllerInfo->FirmwareTurnNumber); - if (ControllerInfo->FirmwareMajorVersion == 6 && - ControllerInfo->FirmwareMinorVersion == 0 && - ControllerInfo->FirmwareTurnNumber < 1) - { - DAC960_Info("FIRMWARE VERSION %s DOES NOT PROVIDE THE CONTROLLER\n", - Controller, Controller->FirmwareVersion); - DAC960_Info("STATUS MONITORING FUNCTIONALITY NEEDED BY THIS DRIVER.\n", - Controller); - DAC960_Info("PLEASE UPGRADE TO VERSION 6.00-01 OR ABOVE.\n", - Controller); - } - /* - Initialize the Controller Channels, Targets, and Memory Size. - */ - Controller->Channels = ControllerInfo->NumberOfPhysicalChannelsPresent; - Controller->Targets = - ControllerInfo->MaximumTargetsPerChannel - [ControllerInfo->NumberOfPhysicalChannelsPresent-1]; - Controller->MemorySize = ControllerInfo->MemorySizeMB; - /* - Initialize the Controller Queue Depth, Driver Queue Depth, Logical Drive - Count, Maximum Blocks per Command, Controller Scatter/Gather Limit, and - Driver Scatter/Gather Limit. The Driver Queue Depth must be at most one - less than the Controller Queue Depth to allow for an automatic drive - rebuild operation. - */ - Controller->ControllerQueueDepth = ControllerInfo->MaximumParallelCommands; - Controller->DriverQueueDepth = Controller->ControllerQueueDepth - 1; - if (Controller->DriverQueueDepth > DAC960_MaxDriverQueueDepth) - Controller->DriverQueueDepth = DAC960_MaxDriverQueueDepth; - Controller->LogicalDriveCount = ControllerInfo->LogicalDevicesPresent; - Controller->MaxBlocksPerCommand = - ControllerInfo->MaximumDataTransferSizeInBlocks; - Controller->ControllerScatterGatherLimit = - ControllerInfo->MaximumScatterGatherEntries; - Controller->DriverScatterGatherLimit = - Controller->ControllerScatterGatherLimit; - if (Controller->DriverScatterGatherLimit > DAC960_V2_ScatterGatherLimit) - Controller->DriverScatterGatherLimit = DAC960_V2_ScatterGatherLimit; - /* - Initialize the Logical Device Information. - */ - while (true) - { - DAC960_V2_LogicalDeviceInfo_T *NewLogicalDeviceInfo = - Controller->V2.NewLogicalDeviceInformation; - DAC960_V2_LogicalDeviceInfo_T *LogicalDeviceInfo; - DAC960_V2_PhysicalDevice_T PhysicalDevice; - - if (!DAC960_V2_NewLogicalDeviceInfo(Controller, LogicalDeviceNumber)) - break; - LogicalDeviceNumber = NewLogicalDeviceInfo->LogicalDeviceNumber; - if (LogicalDeviceNumber >= DAC960_MaxLogicalDrives) { - DAC960_Error("DAC960: Logical Drive Number %d not supported\n", - Controller, LogicalDeviceNumber); - break; - } - if (NewLogicalDeviceInfo->DeviceBlockSizeInBytes != DAC960_BlockSize) { - DAC960_Error("DAC960: Logical Drive Block Size %d not supported\n", - Controller, NewLogicalDeviceInfo->DeviceBlockSizeInBytes); - LogicalDeviceNumber++; - continue; - } - PhysicalDevice.Controller = 0; - PhysicalDevice.Channel = NewLogicalDeviceInfo->Channel; - PhysicalDevice.TargetID = NewLogicalDeviceInfo->TargetID; - PhysicalDevice.LogicalUnit = NewLogicalDeviceInfo->LogicalUnit; - Controller->V2.LogicalDriveToVirtualDevice[LogicalDeviceNumber] = - PhysicalDevice; - if (NewLogicalDeviceInfo->LogicalDeviceState != - DAC960_V2_LogicalDevice_Offline) - Controller->LogicalDriveInitiallyAccessible[LogicalDeviceNumber] = true; - LogicalDeviceInfo = kmalloc(sizeof(DAC960_V2_LogicalDeviceInfo_T), - GFP_ATOMIC); - if (LogicalDeviceInfo == NULL) - return DAC960_Failure(Controller, "LOGICAL DEVICE ALLOCATION"); - Controller->V2.LogicalDeviceInformation[LogicalDeviceNumber] = - LogicalDeviceInfo; - memcpy(LogicalDeviceInfo, NewLogicalDeviceInfo, - sizeof(DAC960_V2_LogicalDeviceInfo_T)); - LogicalDeviceNumber++; - } - return true; -} - - -/* - DAC960_ReportControllerConfiguration reports the Configuration Information - for Controller. -*/ - -static bool DAC960_ReportControllerConfiguration(DAC960_Controller_T - *Controller) -{ - DAC960_Info("Configuring Mylex %s PCI RAID Controller\n", - Controller, Controller->ModelName); - DAC960_Info(" Firmware Version: %s, Channels: %d, Memory Size: %dMB\n", - Controller, Controller->FirmwareVersion, - Controller->Channels, Controller->MemorySize); - DAC960_Info(" PCI Bus: %d, Device: %d, Function: %d, I/O Address: ", - Controller, Controller->Bus, - Controller->Device, Controller->Function); - if (Controller->IO_Address == 0) - DAC960_Info("Unassigned\n", Controller); - else DAC960_Info("0x%X\n", Controller, Controller->IO_Address); - DAC960_Info(" PCI Address: 0x%X mapped at 0x%lX, IRQ Channel: %d\n", - Controller, Controller->PCI_Address, - (unsigned long) Controller->BaseAddress, - Controller->IRQ_Channel); - DAC960_Info(" Controller Queue Depth: %d, " - "Maximum Blocks per Command: %d\n", - Controller, Controller->ControllerQueueDepth, - Controller->MaxBlocksPerCommand); - DAC960_Info(" Driver Queue Depth: %d, " - "Scatter/Gather Limit: %d of %d Segments\n", - Controller, Controller->DriverQueueDepth, - Controller->DriverScatterGatherLimit, - Controller->ControllerScatterGatherLimit); - if (Controller->FirmwareType == DAC960_V1_Controller) - { - DAC960_Info(" Stripe Size: %dKB, Segment Size: %dKB, " - "BIOS Geometry: %d/%d\n", Controller, - Controller->V1.StripeSize, - Controller->V1.SegmentSize, - Controller->V1.GeometryTranslationHeads, - Controller->V1.GeometryTranslationSectors); - if (Controller->V1.SAFTE_EnclosureManagementEnabled) - DAC960_Info(" SAF-TE Enclosure Management Enabled\n", Controller); - } - return true; -} - - -/* - DAC960_V1_ReadDeviceConfiguration reads the Device Configuration Information - for DAC960 V1 Firmware Controllers by requesting the SCSI Inquiry and SCSI - Inquiry Unit Serial Number information for each device connected to - Controller. -*/ - -static bool DAC960_V1_ReadDeviceConfiguration(DAC960_Controller_T - *Controller) -{ - struct dma_loaf local_dma; - - dma_addr_t DCDBs_dma[DAC960_V1_MaxChannels]; - DAC960_V1_DCDB_T *DCDBs_cpu[DAC960_V1_MaxChannels]; - - dma_addr_t SCSI_Inquiry_dma[DAC960_V1_MaxChannels]; - DAC960_SCSI_Inquiry_T *SCSI_Inquiry_cpu[DAC960_V1_MaxChannels]; - - dma_addr_t SCSI_NewInquiryUnitSerialNumberDMA[DAC960_V1_MaxChannels]; - DAC960_SCSI_Inquiry_UnitSerialNumber_T *SCSI_NewInquiryUnitSerialNumberCPU[DAC960_V1_MaxChannels]; - - struct completion Completions[DAC960_V1_MaxChannels]; - unsigned long flags; - int Channel, TargetID; - - if (!init_dma_loaf(Controller->PCIDevice, &local_dma, - DAC960_V1_MaxChannels*(sizeof(DAC960_V1_DCDB_T) + - sizeof(DAC960_SCSI_Inquiry_T) + - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T)))) - return DAC960_Failure(Controller, - "DMA ALLOCATION FAILED IN ReadDeviceConfiguration"); - - for (Channel = 0; Channel < Controller->Channels; Channel++) { - DCDBs_cpu[Channel] = slice_dma_loaf(&local_dma, - sizeof(DAC960_V1_DCDB_T), DCDBs_dma + Channel); - SCSI_Inquiry_cpu[Channel] = slice_dma_loaf(&local_dma, - sizeof(DAC960_SCSI_Inquiry_T), - SCSI_Inquiry_dma + Channel); - SCSI_NewInquiryUnitSerialNumberCPU[Channel] = slice_dma_loaf(&local_dma, - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T), - SCSI_NewInquiryUnitSerialNumberDMA + Channel); - } - - for (TargetID = 0; TargetID < Controller->Targets; TargetID++) - { - /* - * For each channel, submit a probe for a device on that channel. - * The timeout interval for a device that is present is 10 seconds. - * With this approach, the timeout periods can elapse in parallel - * on each channel. - */ - for (Channel = 0; Channel < Controller->Channels; Channel++) - { - dma_addr_t NewInquiryStandardDataDMA = SCSI_Inquiry_dma[Channel]; - DAC960_V1_DCDB_T *DCDB = DCDBs_cpu[Channel]; - dma_addr_t DCDB_dma = DCDBs_dma[Channel]; - DAC960_Command_T *Command = Controller->Commands[Channel]; - struct completion *Completion = &Completions[Channel]; - - init_completion(Completion); - DAC960_V1_ClearCommand(Command); - Command->CommandType = DAC960_ImmediateCommand; - Command->Completion = Completion; - Command->V1.CommandMailbox.Type3.CommandOpcode = DAC960_V1_DCDB; - Command->V1.CommandMailbox.Type3.BusAddress = DCDB_dma; - DCDB->Channel = Channel; - DCDB->TargetID = TargetID; - DCDB->Direction = DAC960_V1_DCDB_DataTransferDeviceToSystem; - DCDB->EarlyStatus = false; - DCDB->Timeout = DAC960_V1_DCDB_Timeout_10_seconds; - DCDB->NoAutomaticRequestSense = false; - DCDB->DisconnectPermitted = true; - DCDB->TransferLength = sizeof(DAC960_SCSI_Inquiry_T); - DCDB->BusAddress = NewInquiryStandardDataDMA; - DCDB->CDBLength = 6; - DCDB->TransferLengthHigh4 = 0; - DCDB->SenseLength = sizeof(DCDB->SenseData); - DCDB->CDB[0] = 0x12; /* INQUIRY */ - DCDB->CDB[1] = 0; /* EVPD = 0 */ - DCDB->CDB[2] = 0; /* Page Code */ - DCDB->CDB[3] = 0; /* Reserved */ - DCDB->CDB[4] = sizeof(DAC960_SCSI_Inquiry_T); - DCDB->CDB[5] = 0; /* Control */ - - spin_lock_irqsave(&Controller->queue_lock, flags); - DAC960_QueueCommand(Command); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - } - /* - * Wait for the problems submitted in the previous loop - * to complete. On the probes that are successful, - * get the serial number of the device that was found. - */ - for (Channel = 0; Channel < Controller->Channels; Channel++) - { - DAC960_SCSI_Inquiry_T *InquiryStandardData = - &Controller->V1.InquiryStandardData[Channel][TargetID]; - DAC960_SCSI_Inquiry_T *NewInquiryStandardData = SCSI_Inquiry_cpu[Channel]; - dma_addr_t NewInquiryUnitSerialNumberDMA = - SCSI_NewInquiryUnitSerialNumberDMA[Channel]; - DAC960_SCSI_Inquiry_UnitSerialNumber_T *NewInquiryUnitSerialNumber = - SCSI_NewInquiryUnitSerialNumberCPU[Channel]; - DAC960_SCSI_Inquiry_UnitSerialNumber_T *InquiryUnitSerialNumber = - &Controller->V1.InquiryUnitSerialNumber[Channel][TargetID]; - DAC960_Command_T *Command = Controller->Commands[Channel]; - DAC960_V1_DCDB_T *DCDB = DCDBs_cpu[Channel]; - struct completion *Completion = &Completions[Channel]; - - wait_for_completion(Completion); - - if (Command->V1.CommandStatus != DAC960_V1_NormalCompletion) { - memset(InquiryStandardData, 0, sizeof(DAC960_SCSI_Inquiry_T)); - InquiryStandardData->PeripheralDeviceType = 0x1F; - continue; - } else - memcpy(InquiryStandardData, NewInquiryStandardData, sizeof(DAC960_SCSI_Inquiry_T)); - - /* Preserve Channel and TargetID values from the previous loop */ - Command->Completion = Completion; - DCDB->TransferLength = sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T); - DCDB->BusAddress = NewInquiryUnitSerialNumberDMA; - DCDB->SenseLength = sizeof(DCDB->SenseData); - DCDB->CDB[0] = 0x12; /* INQUIRY */ - DCDB->CDB[1] = 1; /* EVPD = 1 */ - DCDB->CDB[2] = 0x80; /* Page Code */ - DCDB->CDB[3] = 0; /* Reserved */ - DCDB->CDB[4] = sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T); - DCDB->CDB[5] = 0; /* Control */ - - spin_lock_irqsave(&Controller->queue_lock, flags); - DAC960_QueueCommand(Command); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - wait_for_completion(Completion); - - if (Command->V1.CommandStatus != DAC960_V1_NormalCompletion) { - memset(InquiryUnitSerialNumber, 0, - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T)); - InquiryUnitSerialNumber->PeripheralDeviceType = 0x1F; - } else - memcpy(InquiryUnitSerialNumber, NewInquiryUnitSerialNumber, - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T)); - } - } - free_dma_loaf(Controller->PCIDevice, &local_dma); - return true; -} - - -/* - DAC960_V2_ReadDeviceConfiguration reads the Device Configuration Information - for DAC960 V2 Firmware Controllers by requesting the Physical Device - Information and SCSI Inquiry Unit Serial Number information for each - device connected to Controller. -*/ - -static bool DAC960_V2_ReadDeviceConfiguration(DAC960_Controller_T - *Controller) -{ - unsigned char Channel = 0, TargetID = 0, LogicalUnit = 0; - unsigned short PhysicalDeviceIndex = 0; - - while (true) - { - DAC960_V2_PhysicalDeviceInfo_T *NewPhysicalDeviceInfo = - Controller->V2.NewPhysicalDeviceInformation; - DAC960_V2_PhysicalDeviceInfo_T *PhysicalDeviceInfo; - DAC960_SCSI_Inquiry_UnitSerialNumber_T *NewInquiryUnitSerialNumber = - Controller->V2.NewInquiryUnitSerialNumber; - DAC960_SCSI_Inquiry_UnitSerialNumber_T *InquiryUnitSerialNumber; - - if (!DAC960_V2_NewPhysicalDeviceInfo(Controller, Channel, TargetID, LogicalUnit)) - break; - - PhysicalDeviceInfo = kmalloc(sizeof(DAC960_V2_PhysicalDeviceInfo_T), - GFP_ATOMIC); - if (PhysicalDeviceInfo == NULL) - return DAC960_Failure(Controller, "PHYSICAL DEVICE ALLOCATION"); - Controller->V2.PhysicalDeviceInformation[PhysicalDeviceIndex] = - PhysicalDeviceInfo; - memcpy(PhysicalDeviceInfo, NewPhysicalDeviceInfo, - sizeof(DAC960_V2_PhysicalDeviceInfo_T)); - - InquiryUnitSerialNumber = kmalloc( - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T), GFP_ATOMIC); - if (InquiryUnitSerialNumber == NULL) { - kfree(PhysicalDeviceInfo); - return DAC960_Failure(Controller, "SERIAL NUMBER ALLOCATION"); - } - Controller->V2.InquiryUnitSerialNumber[PhysicalDeviceIndex] = - InquiryUnitSerialNumber; - - Channel = NewPhysicalDeviceInfo->Channel; - TargetID = NewPhysicalDeviceInfo->TargetID; - LogicalUnit = NewPhysicalDeviceInfo->LogicalUnit; - - /* - Some devices do NOT have Unit Serial Numbers. - This command fails for them. But, we still want to - remember those devices are there. Construct a - UnitSerialNumber structure for the failure case. - */ - if (!DAC960_V2_NewInquiryUnitSerialNumber(Controller, Channel, TargetID, LogicalUnit)) { - memset(InquiryUnitSerialNumber, 0, - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T)); - InquiryUnitSerialNumber->PeripheralDeviceType = 0x1F; - } else - memcpy(InquiryUnitSerialNumber, NewInquiryUnitSerialNumber, - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T)); - - PhysicalDeviceIndex++; - LogicalUnit++; - } - return true; -} - - -/* - DAC960_SanitizeInquiryData sanitizes the Vendor, Model, Revision, and - Product Serial Number fields of the Inquiry Standard Data and Inquiry - Unit Serial Number structures. -*/ - -static void DAC960_SanitizeInquiryData(DAC960_SCSI_Inquiry_T - *InquiryStandardData, - DAC960_SCSI_Inquiry_UnitSerialNumber_T - *InquiryUnitSerialNumber, - unsigned char *Vendor, - unsigned char *Model, - unsigned char *Revision, - unsigned char *SerialNumber) -{ - int SerialNumberLength, i; - if (InquiryStandardData->PeripheralDeviceType == 0x1F) return; - for (i = 0; i < sizeof(InquiryStandardData->VendorIdentification); i++) - { - unsigned char VendorCharacter = - InquiryStandardData->VendorIdentification[i]; - Vendor[i] = (VendorCharacter >= ' ' && VendorCharacter <= '~' - ? VendorCharacter : ' '); - } - Vendor[sizeof(InquiryStandardData->VendorIdentification)] = '\0'; - for (i = 0; i < sizeof(InquiryStandardData->ProductIdentification); i++) - { - unsigned char ModelCharacter = - InquiryStandardData->ProductIdentification[i]; - Model[i] = (ModelCharacter >= ' ' && ModelCharacter <= '~' - ? ModelCharacter : ' '); - } - Model[sizeof(InquiryStandardData->ProductIdentification)] = '\0'; - for (i = 0; i < sizeof(InquiryStandardData->ProductRevisionLevel); i++) - { - unsigned char RevisionCharacter = - InquiryStandardData->ProductRevisionLevel[i]; - Revision[i] = (RevisionCharacter >= ' ' && RevisionCharacter <= '~' - ? RevisionCharacter : ' '); - } - Revision[sizeof(InquiryStandardData->ProductRevisionLevel)] = '\0'; - if (InquiryUnitSerialNumber->PeripheralDeviceType == 0x1F) return; - SerialNumberLength = InquiryUnitSerialNumber->PageLength; - if (SerialNumberLength > - sizeof(InquiryUnitSerialNumber->ProductSerialNumber)) - SerialNumberLength = sizeof(InquiryUnitSerialNumber->ProductSerialNumber); - for (i = 0; i < SerialNumberLength; i++) - { - unsigned char SerialNumberCharacter = - InquiryUnitSerialNumber->ProductSerialNumber[i]; - SerialNumber[i] = - (SerialNumberCharacter >= ' ' && SerialNumberCharacter <= '~' - ? SerialNumberCharacter : ' '); - } - SerialNumber[SerialNumberLength] = '\0'; -} - - -/* - DAC960_V1_ReportDeviceConfiguration reports the Device Configuration - Information for DAC960 V1 Firmware Controllers. -*/ - -static bool DAC960_V1_ReportDeviceConfiguration(DAC960_Controller_T - *Controller) -{ - int LogicalDriveNumber, Channel, TargetID; - DAC960_Info(" Physical Devices:\n", Controller); - for (Channel = 0; Channel < Controller->Channels; Channel++) - for (TargetID = 0; TargetID < Controller->Targets; TargetID++) - { - DAC960_SCSI_Inquiry_T *InquiryStandardData = - &Controller->V1.InquiryStandardData[Channel][TargetID]; - DAC960_SCSI_Inquiry_UnitSerialNumber_T *InquiryUnitSerialNumber = - &Controller->V1.InquiryUnitSerialNumber[Channel][TargetID]; - DAC960_V1_DeviceState_T *DeviceState = - &Controller->V1.DeviceState[Channel][TargetID]; - DAC960_V1_ErrorTableEntry_T *ErrorEntry = - &Controller->V1.ErrorTable.ErrorTableEntries[Channel][TargetID]; - char Vendor[1+sizeof(InquiryStandardData->VendorIdentification)]; - char Model[1+sizeof(InquiryStandardData->ProductIdentification)]; - char Revision[1+sizeof(InquiryStandardData->ProductRevisionLevel)]; - char SerialNumber[1+sizeof(InquiryUnitSerialNumber - ->ProductSerialNumber)]; - if (InquiryStandardData->PeripheralDeviceType == 0x1F) continue; - DAC960_SanitizeInquiryData(InquiryStandardData, InquiryUnitSerialNumber, - Vendor, Model, Revision, SerialNumber); - DAC960_Info(" %d:%d%s Vendor: %s Model: %s Revision: %s\n", - Controller, Channel, TargetID, (TargetID < 10 ? " " : ""), - Vendor, Model, Revision); - if (InquiryUnitSerialNumber->PeripheralDeviceType != 0x1F) - DAC960_Info(" Serial Number: %s\n", Controller, SerialNumber); - if (DeviceState->Present && - DeviceState->DeviceType == DAC960_V1_DiskType) - { - if (Controller->V1.DeviceResetCount[Channel][TargetID] > 0) - DAC960_Info(" Disk Status: %s, %u blocks, %d resets\n", - Controller, - (DeviceState->DeviceState == DAC960_V1_Device_Dead - ? "Dead" - : DeviceState->DeviceState - == DAC960_V1_Device_WriteOnly - ? "Write-Only" - : DeviceState->DeviceState - == DAC960_V1_Device_Online - ? "Online" : "Standby"), - DeviceState->DiskSize, - Controller->V1.DeviceResetCount[Channel][TargetID]); - else - DAC960_Info(" Disk Status: %s, %u blocks\n", Controller, - (DeviceState->DeviceState == DAC960_V1_Device_Dead - ? "Dead" - : DeviceState->DeviceState - == DAC960_V1_Device_WriteOnly - ? "Write-Only" - : DeviceState->DeviceState - == DAC960_V1_Device_Online - ? "Online" : "Standby"), - DeviceState->DiskSize); - } - if (ErrorEntry->ParityErrorCount > 0 || - ErrorEntry->SoftErrorCount > 0 || - ErrorEntry->HardErrorCount > 0 || - ErrorEntry->MiscErrorCount > 0) - DAC960_Info(" Errors - Parity: %d, Soft: %d, " - "Hard: %d, Misc: %d\n", Controller, - ErrorEntry->ParityErrorCount, - ErrorEntry->SoftErrorCount, - ErrorEntry->HardErrorCount, - ErrorEntry->MiscErrorCount); - } - DAC960_Info(" Logical Drives:\n", Controller); - for (LogicalDriveNumber = 0; - LogicalDriveNumber < Controller->LogicalDriveCount; - LogicalDriveNumber++) - { - DAC960_V1_LogicalDriveInformation_T *LogicalDriveInformation = - &Controller->V1.LogicalDriveInformation[LogicalDriveNumber]; - DAC960_Info(" /dev/rd/c%dd%d: RAID-%d, %s, %u blocks, %s\n", - Controller, Controller->ControllerNumber, LogicalDriveNumber, - LogicalDriveInformation->RAIDLevel, - (LogicalDriveInformation->LogicalDriveState - == DAC960_V1_LogicalDrive_Online - ? "Online" - : LogicalDriveInformation->LogicalDriveState - == DAC960_V1_LogicalDrive_Critical - ? "Critical" : "Offline"), - LogicalDriveInformation->LogicalDriveSize, - (LogicalDriveInformation->WriteBack - ? "Write Back" : "Write Thru")); - } - return true; -} - - -/* - DAC960_V2_ReportDeviceConfiguration reports the Device Configuration - Information for DAC960 V2 Firmware Controllers. -*/ - -static bool DAC960_V2_ReportDeviceConfiguration(DAC960_Controller_T - *Controller) -{ - int PhysicalDeviceIndex, LogicalDriveNumber; - DAC960_Info(" Physical Devices:\n", Controller); - for (PhysicalDeviceIndex = 0; - PhysicalDeviceIndex < DAC960_V2_MaxPhysicalDevices; - PhysicalDeviceIndex++) - { - DAC960_V2_PhysicalDeviceInfo_T *PhysicalDeviceInfo = - Controller->V2.PhysicalDeviceInformation[PhysicalDeviceIndex]; - DAC960_SCSI_Inquiry_T *InquiryStandardData = - (DAC960_SCSI_Inquiry_T *) &PhysicalDeviceInfo->SCSI_InquiryData; - DAC960_SCSI_Inquiry_UnitSerialNumber_T *InquiryUnitSerialNumber = - Controller->V2.InquiryUnitSerialNumber[PhysicalDeviceIndex]; - char Vendor[1+sizeof(InquiryStandardData->VendorIdentification)]; - char Model[1+sizeof(InquiryStandardData->ProductIdentification)]; - char Revision[1+sizeof(InquiryStandardData->ProductRevisionLevel)]; - char SerialNumber[1+sizeof(InquiryUnitSerialNumber->ProductSerialNumber)]; - if (PhysicalDeviceInfo == NULL) break; - DAC960_SanitizeInquiryData(InquiryStandardData, InquiryUnitSerialNumber, - Vendor, Model, Revision, SerialNumber); - DAC960_Info(" %d:%d%s Vendor: %s Model: %s Revision: %s\n", - Controller, - PhysicalDeviceInfo->Channel, - PhysicalDeviceInfo->TargetID, - (PhysicalDeviceInfo->TargetID < 10 ? " " : ""), - Vendor, Model, Revision); - if (PhysicalDeviceInfo->NegotiatedSynchronousMegaTransfers == 0) - DAC960_Info(" %sAsynchronous\n", Controller, - (PhysicalDeviceInfo->NegotiatedDataWidthBits == 16 - ? "Wide " :"")); - else - DAC960_Info(" %sSynchronous at %d MB/sec\n", Controller, - (PhysicalDeviceInfo->NegotiatedDataWidthBits == 16 - ? "Wide " :""), - (PhysicalDeviceInfo->NegotiatedSynchronousMegaTransfers - * PhysicalDeviceInfo->NegotiatedDataWidthBits/8)); - if (InquiryUnitSerialNumber->PeripheralDeviceType != 0x1F) - DAC960_Info(" Serial Number: %s\n", Controller, SerialNumber); - if (PhysicalDeviceInfo->PhysicalDeviceState == - DAC960_V2_Device_Unconfigured) - continue; - DAC960_Info(" Disk Status: %s, %u blocks\n", Controller, - (PhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_Online - ? "Online" - : PhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_Rebuild - ? "Rebuild" - : PhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_Missing - ? "Missing" - : PhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_Critical - ? "Critical" - : PhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_Dead - ? "Dead" - : PhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_SuspectedDead - ? "Suspected-Dead" - : PhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_CommandedOffline - ? "Commanded-Offline" - : PhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_Standby - ? "Standby" : "Unknown"), - PhysicalDeviceInfo->ConfigurableDeviceSize); - if (PhysicalDeviceInfo->ParityErrors == 0 && - PhysicalDeviceInfo->SoftErrors == 0 && - PhysicalDeviceInfo->HardErrors == 0 && - PhysicalDeviceInfo->MiscellaneousErrors == 0 && - PhysicalDeviceInfo->CommandTimeouts == 0 && - PhysicalDeviceInfo->Retries == 0 && - PhysicalDeviceInfo->Aborts == 0 && - PhysicalDeviceInfo->PredictedFailuresDetected == 0) - continue; - DAC960_Info(" Errors - Parity: %d, Soft: %d, " - "Hard: %d, Misc: %d\n", Controller, - PhysicalDeviceInfo->ParityErrors, - PhysicalDeviceInfo->SoftErrors, - PhysicalDeviceInfo->HardErrors, - PhysicalDeviceInfo->MiscellaneousErrors); - DAC960_Info(" Timeouts: %d, Retries: %d, " - "Aborts: %d, Predicted: %d\n", Controller, - PhysicalDeviceInfo->CommandTimeouts, - PhysicalDeviceInfo->Retries, - PhysicalDeviceInfo->Aborts, - PhysicalDeviceInfo->PredictedFailuresDetected); - } - DAC960_Info(" Logical Drives:\n", Controller); - for (LogicalDriveNumber = 0; - LogicalDriveNumber < DAC960_MaxLogicalDrives; - LogicalDriveNumber++) - { - DAC960_V2_LogicalDeviceInfo_T *LogicalDeviceInfo = - Controller->V2.LogicalDeviceInformation[LogicalDriveNumber]; - static const unsigned char *ReadCacheStatus[] = { - "Read Cache Disabled", - "Read Cache Enabled", - "Read Ahead Enabled", - "Intelligent Read Ahead Enabled", - "-", "-", "-", "-" - }; - static const unsigned char *WriteCacheStatus[] = { - "Write Cache Disabled", - "Logical Device Read Only", - "Write Cache Enabled", - "Intelligent Write Cache Enabled", - "-", "-", "-", "-" - }; - unsigned char *GeometryTranslation; - if (LogicalDeviceInfo == NULL) continue; - switch (LogicalDeviceInfo->DriveGeometry) - { - case DAC960_V2_Geometry_128_32: - GeometryTranslation = "128/32"; - break; - case DAC960_V2_Geometry_255_63: - GeometryTranslation = "255/63"; - break; - default: - GeometryTranslation = "Invalid"; - DAC960_Error("Illegal Logical Device Geometry %d\n", - Controller, LogicalDeviceInfo->DriveGeometry); - break; - } - DAC960_Info(" /dev/rd/c%dd%d: RAID-%d, %s, %u blocks\n", - Controller, Controller->ControllerNumber, LogicalDriveNumber, - LogicalDeviceInfo->RAIDLevel, - (LogicalDeviceInfo->LogicalDeviceState - == DAC960_V2_LogicalDevice_Online - ? "Online" - : LogicalDeviceInfo->LogicalDeviceState - == DAC960_V2_LogicalDevice_Critical - ? "Critical" : "Offline"), - LogicalDeviceInfo->ConfigurableDeviceSize); - DAC960_Info(" Logical Device %s, BIOS Geometry: %s\n", - Controller, - (LogicalDeviceInfo->LogicalDeviceControl - .LogicalDeviceInitialized - ? "Initialized" : "Uninitialized"), - GeometryTranslation); - if (LogicalDeviceInfo->StripeSize == 0) - { - if (LogicalDeviceInfo->CacheLineSize == 0) - DAC960_Info(" Stripe Size: N/A, " - "Segment Size: N/A\n", Controller); - else - DAC960_Info(" Stripe Size: N/A, " - "Segment Size: %dKB\n", Controller, - 1 << (LogicalDeviceInfo->CacheLineSize - 2)); - } - else - { - if (LogicalDeviceInfo->CacheLineSize == 0) - DAC960_Info(" Stripe Size: %dKB, " - "Segment Size: N/A\n", Controller, - 1 << (LogicalDeviceInfo->StripeSize - 2)); - else - DAC960_Info(" Stripe Size: %dKB, " - "Segment Size: %dKB\n", Controller, - 1 << (LogicalDeviceInfo->StripeSize - 2), - 1 << (LogicalDeviceInfo->CacheLineSize - 2)); - } - DAC960_Info(" %s, %s\n", Controller, - ReadCacheStatus[ - LogicalDeviceInfo->LogicalDeviceControl.ReadCache], - WriteCacheStatus[ - LogicalDeviceInfo->LogicalDeviceControl.WriteCache]); - if (LogicalDeviceInfo->SoftErrors > 0 || - LogicalDeviceInfo->CommandsFailed > 0 || - LogicalDeviceInfo->DeferredWriteErrors) - DAC960_Info(" Errors - Soft: %d, Failed: %d, " - "Deferred Write: %d\n", Controller, - LogicalDeviceInfo->SoftErrors, - LogicalDeviceInfo->CommandsFailed, - LogicalDeviceInfo->DeferredWriteErrors); - - } - return true; -} - -/* - DAC960_RegisterBlockDevice registers the Block Device structures - associated with Controller. -*/ - -static bool DAC960_RegisterBlockDevice(DAC960_Controller_T *Controller) -{ - int MajorNumber = DAC960_MAJOR + Controller->ControllerNumber; - int n; - - /* - Register the Block Device Major Number for this DAC960 Controller. - */ - if (register_blkdev(MajorNumber, "dac960") < 0) - return false; - - for (n = 0; n < DAC960_MaxLogicalDrives; n++) { - struct gendisk *disk = Controller->disks[n]; - struct request_queue *RequestQueue; - - /* for now, let all request queues share controller's lock */ - RequestQueue = blk_init_queue(DAC960_RequestFunction,&Controller->queue_lock); - if (!RequestQueue) { - printk("DAC960: failure to allocate request queue\n"); - continue; - } - Controller->RequestQueue[n] = RequestQueue; - RequestQueue->queuedata = Controller; - blk_queue_max_segments(RequestQueue, Controller->DriverScatterGatherLimit); - blk_queue_max_hw_sectors(RequestQueue, Controller->MaxBlocksPerCommand); - disk->queue = RequestQueue; - sprintf(disk->disk_name, "rd/c%dd%d", Controller->ControllerNumber, n); - disk->major = MajorNumber; - disk->first_minor = n << DAC960_MaxPartitionsBits; - disk->fops = &DAC960_BlockDeviceOperations; - } - /* - Indicate the Block Device Registration completed successfully, - */ - return true; -} - - -/* - DAC960_UnregisterBlockDevice unregisters the Block Device structures - associated with Controller. -*/ - -static void DAC960_UnregisterBlockDevice(DAC960_Controller_T *Controller) -{ - int MajorNumber = DAC960_MAJOR + Controller->ControllerNumber; - int disk; - - /* does order matter when deleting gendisk and cleanup in request queue? */ - for (disk = 0; disk < DAC960_MaxLogicalDrives; disk++) { - del_gendisk(Controller->disks[disk]); - blk_cleanup_queue(Controller->RequestQueue[disk]); - Controller->RequestQueue[disk] = NULL; - } - - /* - Unregister the Block Device Major Number for this DAC960 Controller. - */ - unregister_blkdev(MajorNumber, "dac960"); -} - -/* - DAC960_ComputeGenericDiskInfo computes the values for the Generic Disk - Information Partition Sector Counts and Block Sizes. -*/ - -static void DAC960_ComputeGenericDiskInfo(DAC960_Controller_T *Controller) -{ - int disk; - for (disk = 0; disk < DAC960_MaxLogicalDrives; disk++) - set_capacity(Controller->disks[disk], disk_size(Controller, disk)); -} - -/* - DAC960_ReportErrorStatus reports Controller BIOS Messages passed through - the Error Status Register when the driver performs the BIOS handshaking. - It returns true for fatal errors and false otherwise. -*/ - -static bool DAC960_ReportErrorStatus(DAC960_Controller_T *Controller, - unsigned char ErrorStatus, - unsigned char Parameter0, - unsigned char Parameter1) -{ - switch (ErrorStatus) - { - case 0x00: - DAC960_Notice("Physical Device %d:%d Not Responding\n", - Controller, Parameter1, Parameter0); - break; - case 0x08: - if (Controller->DriveSpinUpMessageDisplayed) break; - DAC960_Notice("Spinning Up Drives\n", Controller); - Controller->DriveSpinUpMessageDisplayed = true; - break; - case 0x30: - DAC960_Notice("Configuration Checksum Error\n", Controller); - break; - case 0x60: - DAC960_Notice("Mirror Race Recovery Failed\n", Controller); - break; - case 0x70: - DAC960_Notice("Mirror Race Recovery In Progress\n", Controller); - break; - case 0x90: - DAC960_Notice("Physical Device %d:%d COD Mismatch\n", - Controller, Parameter1, Parameter0); - break; - case 0xA0: - DAC960_Notice("Logical Drive Installation Aborted\n", Controller); - break; - case 0xB0: - DAC960_Notice("Mirror Race On A Critical Logical Drive\n", Controller); - break; - case 0xD0: - DAC960_Notice("New Controller Configuration Found\n", Controller); - break; - case 0xF0: - DAC960_Error("Fatal Memory Parity Error for Controller at\n", Controller); - return true; - default: - DAC960_Error("Unknown Initialization Error %02X for Controller at\n", - Controller, ErrorStatus); - return true; - } - return false; -} - - -/* - * DAC960_DetectCleanup releases the resources that were allocated - * during DAC960_DetectController(). DAC960_DetectController can - * has several internal failure points, so not ALL resources may - * have been allocated. It's important to free only - * resources that HAVE been allocated. The code below always - * tests that the resource has been allocated before attempting to - * free it. - */ -static void DAC960_DetectCleanup(DAC960_Controller_T *Controller) -{ - int i; - - /* Free the memory mailbox, status, and related structures */ - free_dma_loaf(Controller->PCIDevice, &Controller->DmaPages); - if (Controller->MemoryMappedAddress) { - switch(Controller->HardwareType) - { - case DAC960_GEM_Controller: - DAC960_GEM_DisableInterrupts(Controller->BaseAddress); - break; - case DAC960_BA_Controller: - DAC960_BA_DisableInterrupts(Controller->BaseAddress); - break; - case DAC960_LP_Controller: - DAC960_LP_DisableInterrupts(Controller->BaseAddress); - break; - case DAC960_LA_Controller: - DAC960_LA_DisableInterrupts(Controller->BaseAddress); - break; - case DAC960_PG_Controller: - DAC960_PG_DisableInterrupts(Controller->BaseAddress); - break; - case DAC960_PD_Controller: - DAC960_PD_DisableInterrupts(Controller->BaseAddress); - break; - case DAC960_P_Controller: - DAC960_PD_DisableInterrupts(Controller->BaseAddress); - break; - } - iounmap(Controller->MemoryMappedAddress); - } - if (Controller->IRQ_Channel) - free_irq(Controller->IRQ_Channel, Controller); - if (Controller->IO_Address) - release_region(Controller->IO_Address, 0x80); - pci_disable_device(Controller->PCIDevice); - for (i = 0; (i < DAC960_MaxLogicalDrives) && Controller->disks[i]; i++) - put_disk(Controller->disks[i]); - DAC960_Controllers[Controller->ControllerNumber] = NULL; - kfree(Controller); -} - - -/* - DAC960_DetectController detects Mylex DAC960/AcceleRAID/eXtremeRAID - PCI RAID Controllers by interrogating the PCI Configuration Space for - Controller Type. -*/ - -static DAC960_Controller_T * -DAC960_DetectController(struct pci_dev *PCI_Device, - const struct pci_device_id *entry) -{ - struct DAC960_privdata *privdata = - (struct DAC960_privdata *)entry->driver_data; - irq_handler_t InterruptHandler = privdata->InterruptHandler; - unsigned int MemoryWindowSize = privdata->MemoryWindowSize; - DAC960_Controller_T *Controller = NULL; - unsigned char DeviceFunction = PCI_Device->devfn; - unsigned char ErrorStatus, Parameter0, Parameter1; - unsigned int IRQ_Channel; - void __iomem *BaseAddress; - int i; - - Controller = kzalloc(sizeof(DAC960_Controller_T), GFP_ATOMIC); - if (Controller == NULL) { - DAC960_Error("Unable to allocate Controller structure for " - "Controller at\n", NULL); - return NULL; - } - Controller->ControllerNumber = DAC960_ControllerCount; - DAC960_Controllers[DAC960_ControllerCount++] = Controller; - Controller->Bus = PCI_Device->bus->number; - Controller->FirmwareType = privdata->FirmwareType; - Controller->HardwareType = privdata->HardwareType; - Controller->Device = DeviceFunction >> 3; - Controller->Function = DeviceFunction & 0x7; - Controller->PCIDevice = PCI_Device; - strcpy(Controller->FullModelName, "DAC960"); - - if (pci_enable_device(PCI_Device)) - goto Failure; - - switch (Controller->HardwareType) - { - case DAC960_GEM_Controller: - Controller->PCI_Address = pci_resource_start(PCI_Device, 0); - break; - case DAC960_BA_Controller: - Controller->PCI_Address = pci_resource_start(PCI_Device, 0); - break; - case DAC960_LP_Controller: - Controller->PCI_Address = pci_resource_start(PCI_Device, 0); - break; - case DAC960_LA_Controller: - Controller->PCI_Address = pci_resource_start(PCI_Device, 0); - break; - case DAC960_PG_Controller: - Controller->PCI_Address = pci_resource_start(PCI_Device, 0); - break; - case DAC960_PD_Controller: - Controller->IO_Address = pci_resource_start(PCI_Device, 0); - Controller->PCI_Address = pci_resource_start(PCI_Device, 1); - break; - case DAC960_P_Controller: - Controller->IO_Address = pci_resource_start(PCI_Device, 0); - Controller->PCI_Address = pci_resource_start(PCI_Device, 1); - break; - } - - pci_set_drvdata(PCI_Device, (void *)((long)Controller->ControllerNumber)); - for (i = 0; i < DAC960_MaxLogicalDrives; i++) { - Controller->disks[i] = alloc_disk(1<<DAC960_MaxPartitionsBits); - if (!Controller->disks[i]) - goto Failure; - Controller->disks[i]->private_data = (void *)((long)i); - } - init_waitqueue_head(&Controller->CommandWaitQueue); - init_waitqueue_head(&Controller->HealthStatusWaitQueue); - spin_lock_init(&Controller->queue_lock); - DAC960_AnnounceDriver(Controller); - /* - Map the Controller Register Window. - */ - if (MemoryWindowSize < PAGE_SIZE) - MemoryWindowSize = PAGE_SIZE; - Controller->MemoryMappedAddress = - ioremap_nocache(Controller->PCI_Address & PAGE_MASK, MemoryWindowSize); - Controller->BaseAddress = - Controller->MemoryMappedAddress + (Controller->PCI_Address & ~PAGE_MASK); - if (Controller->MemoryMappedAddress == NULL) - { - DAC960_Error("Unable to map Controller Register Window for " - "Controller at\n", Controller); - goto Failure; - } - BaseAddress = Controller->BaseAddress; - switch (Controller->HardwareType) - { - case DAC960_GEM_Controller: - DAC960_GEM_DisableInterrupts(BaseAddress); - DAC960_GEM_AcknowledgeHardwareMailboxStatus(BaseAddress); - udelay(1000); - while (DAC960_GEM_InitializationInProgressP(BaseAddress)) - { - if (DAC960_GEM_ReadErrorStatus(BaseAddress, &ErrorStatus, - &Parameter0, &Parameter1) && - DAC960_ReportErrorStatus(Controller, ErrorStatus, - Parameter0, Parameter1)) - goto Failure; - udelay(10); - } - if (!DAC960_V2_EnableMemoryMailboxInterface(Controller)) - { - DAC960_Error("Unable to Enable Memory Mailbox Interface " - "for Controller at\n", Controller); - goto Failure; - } - DAC960_GEM_EnableInterrupts(BaseAddress); - Controller->QueueCommand = DAC960_GEM_QueueCommand; - Controller->ReadControllerConfiguration = - DAC960_V2_ReadControllerConfiguration; - Controller->ReadDeviceConfiguration = - DAC960_V2_ReadDeviceConfiguration; - Controller->ReportDeviceConfiguration = - DAC960_V2_ReportDeviceConfiguration; - Controller->QueueReadWriteCommand = - DAC960_V2_QueueReadWriteCommand; - break; - case DAC960_BA_Controller: - DAC960_BA_DisableInterrupts(BaseAddress); - DAC960_BA_AcknowledgeHardwareMailboxStatus(BaseAddress); - udelay(1000); - while (DAC960_BA_InitializationInProgressP(BaseAddress)) - { - if (DAC960_BA_ReadErrorStatus(BaseAddress, &ErrorStatus, - &Parameter0, &Parameter1) && - DAC960_ReportErrorStatus(Controller, ErrorStatus, - Parameter0, Parameter1)) - goto Failure; - udelay(10); - } - if (!DAC960_V2_EnableMemoryMailboxInterface(Controller)) - { - DAC960_Error("Unable to Enable Memory Mailbox Interface " - "for Controller at\n", Controller); - goto Failure; - } - DAC960_BA_EnableInterrupts(BaseAddress); - Controller->QueueCommand = DAC960_BA_QueueCommand; - Controller->ReadControllerConfiguration = - DAC960_V2_ReadControllerConfiguration; - Controller->ReadDeviceConfiguration = - DAC960_V2_ReadDeviceConfiguration; - Controller->ReportDeviceConfiguration = - DAC960_V2_ReportDeviceConfiguration; - Controller->QueueReadWriteCommand = - DAC960_V2_QueueReadWriteCommand; - break; - case DAC960_LP_Controller: - DAC960_LP_DisableInterrupts(BaseAddress); - DAC960_LP_AcknowledgeHardwareMailboxStatus(BaseAddress); - udelay(1000); - while (DAC960_LP_InitializationInProgressP(BaseAddress)) - { - if (DAC960_LP_ReadErrorStatus(BaseAddress, &ErrorStatus, - &Parameter0, &Parameter1) && - DAC960_ReportErrorStatus(Controller, ErrorStatus, - Parameter0, Parameter1)) - goto Failure; - udelay(10); - } - if (!DAC960_V2_EnableMemoryMailboxInterface(Controller)) - { - DAC960_Error("Unable to Enable Memory Mailbox Interface " - "for Controller at\n", Controller); - goto Failure; - } - DAC960_LP_EnableInterrupts(BaseAddress); - Controller->QueueCommand = DAC960_LP_QueueCommand; - Controller->ReadControllerConfiguration = - DAC960_V2_ReadControllerConfiguration; - Controller->ReadDeviceConfiguration = - DAC960_V2_ReadDeviceConfiguration; - Controller->ReportDeviceConfiguration = - DAC960_V2_ReportDeviceConfiguration; - Controller->QueueReadWriteCommand = - DAC960_V2_QueueReadWriteCommand; - break; - case DAC960_LA_Controller: - DAC960_LA_DisableInterrupts(BaseAddress); - DAC960_LA_AcknowledgeHardwareMailboxStatus(BaseAddress); - udelay(1000); - while (DAC960_LA_InitializationInProgressP(BaseAddress)) - { - if (DAC960_LA_ReadErrorStatus(BaseAddress, &ErrorStatus, - &Parameter0, &Parameter1) && - DAC960_ReportErrorStatus(Controller, ErrorStatus, - Parameter0, Parameter1)) - goto Failure; - udelay(10); - } - if (!DAC960_V1_EnableMemoryMailboxInterface(Controller)) - { - DAC960_Error("Unable to Enable Memory Mailbox Interface " - "for Controller at\n", Controller); - goto Failure; - } - DAC960_LA_EnableInterrupts(BaseAddress); - if (Controller->V1.DualModeMemoryMailboxInterface) - Controller->QueueCommand = DAC960_LA_QueueCommandDualMode; - else Controller->QueueCommand = DAC960_LA_QueueCommandSingleMode; - Controller->ReadControllerConfiguration = - DAC960_V1_ReadControllerConfiguration; - Controller->ReadDeviceConfiguration = - DAC960_V1_ReadDeviceConfiguration; - Controller->ReportDeviceConfiguration = - DAC960_V1_ReportDeviceConfiguration; - Controller->QueueReadWriteCommand = - DAC960_V1_QueueReadWriteCommand; - break; - case DAC960_PG_Controller: - DAC960_PG_DisableInterrupts(BaseAddress); - DAC960_PG_AcknowledgeHardwareMailboxStatus(BaseAddress); - udelay(1000); - while (DAC960_PG_InitializationInProgressP(BaseAddress)) - { - if (DAC960_PG_ReadErrorStatus(BaseAddress, &ErrorStatus, - &Parameter0, &Parameter1) && - DAC960_ReportErrorStatus(Controller, ErrorStatus, - Parameter0, Parameter1)) - goto Failure; - udelay(10); - } - if (!DAC960_V1_EnableMemoryMailboxInterface(Controller)) - { - DAC960_Error("Unable to Enable Memory Mailbox Interface " - "for Controller at\n", Controller); - goto Failure; - } - DAC960_PG_EnableInterrupts(BaseAddress); - if (Controller->V1.DualModeMemoryMailboxInterface) - Controller->QueueCommand = DAC960_PG_QueueCommandDualMode; - else Controller->QueueCommand = DAC960_PG_QueueCommandSingleMode; - Controller->ReadControllerConfiguration = - DAC960_V1_ReadControllerConfiguration; - Controller->ReadDeviceConfiguration = - DAC960_V1_ReadDeviceConfiguration; - Controller->ReportDeviceConfiguration = - DAC960_V1_ReportDeviceConfiguration; - Controller->QueueReadWriteCommand = - DAC960_V1_QueueReadWriteCommand; - break; - case DAC960_PD_Controller: - if (!request_region(Controller->IO_Address, 0x80, - Controller->FullModelName)) { - DAC960_Error("IO port 0x%lx busy for Controller at\n", - Controller, Controller->IO_Address); - goto Failure; - } - DAC960_PD_DisableInterrupts(BaseAddress); - DAC960_PD_AcknowledgeStatus(BaseAddress); - udelay(1000); - while (DAC960_PD_InitializationInProgressP(BaseAddress)) - { - if (DAC960_PD_ReadErrorStatus(BaseAddress, &ErrorStatus, - &Parameter0, &Parameter1) && - DAC960_ReportErrorStatus(Controller, ErrorStatus, - Parameter0, Parameter1)) - goto Failure; - udelay(10); - } - if (!DAC960_V1_EnableMemoryMailboxInterface(Controller)) - { - DAC960_Error("Unable to allocate DMA mapped memory " - "for Controller at\n", Controller); - goto Failure; - } - DAC960_PD_EnableInterrupts(BaseAddress); - Controller->QueueCommand = DAC960_PD_QueueCommand; - Controller->ReadControllerConfiguration = - DAC960_V1_ReadControllerConfiguration; - Controller->ReadDeviceConfiguration = - DAC960_V1_ReadDeviceConfiguration; - Controller->ReportDeviceConfiguration = - DAC960_V1_ReportDeviceConfiguration; - Controller->QueueReadWriteCommand = - DAC960_V1_QueueReadWriteCommand; - break; - case DAC960_P_Controller: - if (!request_region(Controller->IO_Address, 0x80, - Controller->FullModelName)){ - DAC960_Error("IO port 0x%lx busy for Controller at\n", - Controller, Controller->IO_Address); - goto Failure; - } - DAC960_PD_DisableInterrupts(BaseAddress); - DAC960_PD_AcknowledgeStatus(BaseAddress); - udelay(1000); - while (DAC960_PD_InitializationInProgressP(BaseAddress)) - { - if (DAC960_PD_ReadErrorStatus(BaseAddress, &ErrorStatus, - &Parameter0, &Parameter1) && - DAC960_ReportErrorStatus(Controller, ErrorStatus, - Parameter0, Parameter1)) - goto Failure; - udelay(10); - } - if (!DAC960_V1_EnableMemoryMailboxInterface(Controller)) - { - DAC960_Error("Unable to allocate DMA mapped memory" - "for Controller at\n", Controller); - goto Failure; - } - DAC960_PD_EnableInterrupts(BaseAddress); - Controller->QueueCommand = DAC960_P_QueueCommand; - Controller->ReadControllerConfiguration = - DAC960_V1_ReadControllerConfiguration; - Controller->ReadDeviceConfiguration = - DAC960_V1_ReadDeviceConfiguration; - Controller->ReportDeviceConfiguration = - DAC960_V1_ReportDeviceConfiguration; - Controller->QueueReadWriteCommand = - DAC960_V1_QueueReadWriteCommand; - break; - } - /* - Acquire shared access to the IRQ Channel. - */ - IRQ_Channel = PCI_Device->irq; - if (request_irq(IRQ_Channel, InterruptHandler, IRQF_SHARED, - Controller->FullModelName, Controller) < 0) - { - DAC960_Error("Unable to acquire IRQ Channel %d for Controller at\n", - Controller, Controller->IRQ_Channel); - goto Failure; - } - Controller->IRQ_Channel = IRQ_Channel; - Controller->InitialCommand.CommandIdentifier = 1; - Controller->InitialCommand.Controller = Controller; - Controller->Commands[0] = &Controller->InitialCommand; - Controller->FreeCommands = &Controller->InitialCommand; - return Controller; - -Failure: - if (Controller->IO_Address == 0) - DAC960_Error("PCI Bus %d Device %d Function %d I/O Address N/A " - "PCI Address 0x%X\n", Controller, - Controller->Bus, Controller->Device, - Controller->Function, Controller->PCI_Address); - else - DAC960_Error("PCI Bus %d Device %d Function %d I/O Address " - "0x%X PCI Address 0x%X\n", Controller, - Controller->Bus, Controller->Device, - Controller->Function, Controller->IO_Address, - Controller->PCI_Address); - DAC960_DetectCleanup(Controller); - DAC960_ControllerCount--; - return NULL; -} - -/* - DAC960_InitializeController initializes Controller. -*/ - -static bool -DAC960_InitializeController(DAC960_Controller_T *Controller) -{ - if (DAC960_ReadControllerConfiguration(Controller) && - DAC960_ReportControllerConfiguration(Controller) && - DAC960_CreateAuxiliaryStructures(Controller) && - DAC960_ReadDeviceConfiguration(Controller) && - DAC960_ReportDeviceConfiguration(Controller) && - DAC960_RegisterBlockDevice(Controller)) - { - /* - Initialize the Monitoring Timer. - */ - timer_setup(&Controller->MonitoringTimer, - DAC960_MonitoringTimerFunction, 0); - Controller->MonitoringTimer.expires = - jiffies + DAC960_MonitoringTimerInterval; - add_timer(&Controller->MonitoringTimer); - Controller->ControllerInitialized = true; - return true; - } - return false; -} - - -/* - DAC960_FinalizeController finalizes Controller. -*/ - -static void DAC960_FinalizeController(DAC960_Controller_T *Controller) -{ - if (Controller->ControllerInitialized) - { - unsigned long flags; - - /* - * Acquiring and releasing lock here eliminates - * a very low probability race. - * - * The code below allocates controller command structures - * from the free list without holding the controller lock. - * This is safe assuming there is no other activity on - * the controller at the time. - * - * But, there might be a monitoring command still - * in progress. Setting the Shutdown flag while holding - * the lock ensures that there is no monitoring command - * in the interrupt handler currently, and any monitoring - * commands that complete from this time on will NOT return - * their command structure to the free list. - */ - - spin_lock_irqsave(&Controller->queue_lock, flags); - Controller->ShutdownMonitoringTimer = 1; - spin_unlock_irqrestore(&Controller->queue_lock, flags); - - del_timer_sync(&Controller->MonitoringTimer); - if (Controller->FirmwareType == DAC960_V1_Controller) - { - DAC960_Notice("Flushing Cache...", Controller); - DAC960_V1_ExecuteType3(Controller, DAC960_V1_Flush, 0); - DAC960_Notice("done\n", Controller); - - if (Controller->HardwareType == DAC960_PD_Controller) - release_region(Controller->IO_Address, 0x80); - } - else - { - DAC960_Notice("Flushing Cache...", Controller); - DAC960_V2_DeviceOperation(Controller, DAC960_V2_PauseDevice, - DAC960_V2_RAID_Controller); - DAC960_Notice("done\n", Controller); - } - } - DAC960_UnregisterBlockDevice(Controller); - DAC960_DestroyAuxiliaryStructures(Controller); - DAC960_DestroyProcEntries(Controller); - DAC960_DetectCleanup(Controller); -} - - -/* - DAC960_Probe verifies controller's existence and - initializes the DAC960 Driver for that controller. -*/ - -static int -DAC960_Probe(struct pci_dev *dev, const struct pci_device_id *entry) -{ - int disk; - DAC960_Controller_T *Controller; - - if (DAC960_ControllerCount == DAC960_MaxControllers) - { - DAC960_Error("More than %d DAC960 Controllers detected - " - "ignoring from Controller at\n", - NULL, DAC960_MaxControllers); - return -ENODEV; - } - - Controller = DAC960_DetectController(dev, entry); - if (!Controller) - return -ENODEV; - - if (!DAC960_InitializeController(Controller)) { - DAC960_FinalizeController(Controller); - return -ENODEV; - } - - for (disk = 0; disk < DAC960_MaxLogicalDrives; disk++) { - set_capacity(Controller->disks[disk], disk_size(Controller, disk)); - add_disk(Controller->disks[disk]); - } - DAC960_CreateProcEntries(Controller); - return 0; -} - - -/* - DAC960_Finalize finalizes the DAC960 Driver. -*/ - -static void DAC960_Remove(struct pci_dev *PCI_Device) -{ - int Controller_Number = (long)pci_get_drvdata(PCI_Device); - DAC960_Controller_T *Controller = DAC960_Controllers[Controller_Number]; - if (Controller != NULL) - DAC960_FinalizeController(Controller); -} - - -/* - DAC960_V1_QueueReadWriteCommand prepares and queues a Read/Write Command for - DAC960 V1 Firmware Controllers. -*/ - -static void DAC960_V1_QueueReadWriteCommand(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - DAC960_V1_CommandMailbox_T *CommandMailbox = &Command->V1.CommandMailbox; - DAC960_V1_ScatterGatherSegment_T *ScatterGatherList = - Command->V1.ScatterGatherList; - struct scatterlist *ScatterList = Command->V1.ScatterList; - - DAC960_V1_ClearCommand(Command); - - if (Command->SegmentCount == 1) - { - if (Command->DmaDirection == PCI_DMA_FROMDEVICE) - CommandMailbox->Type5.CommandOpcode = DAC960_V1_Read; - else - CommandMailbox->Type5.CommandOpcode = DAC960_V1_Write; - - CommandMailbox->Type5.LD.TransferLength = Command->BlockCount; - CommandMailbox->Type5.LD.LogicalDriveNumber = Command->LogicalDriveNumber; - CommandMailbox->Type5.LogicalBlockAddress = Command->BlockNumber; - CommandMailbox->Type5.BusAddress = - (DAC960_BusAddress32_T)sg_dma_address(ScatterList); - } - else - { - int i; - - if (Command->DmaDirection == PCI_DMA_FROMDEVICE) - CommandMailbox->Type5.CommandOpcode = DAC960_V1_ReadWithScatterGather; - else - CommandMailbox->Type5.CommandOpcode = DAC960_V1_WriteWithScatterGather; - - CommandMailbox->Type5.LD.TransferLength = Command->BlockCount; - CommandMailbox->Type5.LD.LogicalDriveNumber = Command->LogicalDriveNumber; - CommandMailbox->Type5.LogicalBlockAddress = Command->BlockNumber; - CommandMailbox->Type5.BusAddress = Command->V1.ScatterGatherListDMA; - - CommandMailbox->Type5.ScatterGatherCount = Command->SegmentCount; - - for (i = 0; i < Command->SegmentCount; i++, ScatterList++, ScatterGatherList++) { - ScatterGatherList->SegmentDataPointer = - (DAC960_BusAddress32_T)sg_dma_address(ScatterList); - ScatterGatherList->SegmentByteCount = - (DAC960_ByteCount32_T)sg_dma_len(ScatterList); - } - } - DAC960_QueueCommand(Command); -} - - -/* - DAC960_V2_QueueReadWriteCommand prepares and queues a Read/Write Command for - DAC960 V2 Firmware Controllers. -*/ - -static void DAC960_V2_QueueReadWriteCommand(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - DAC960_V2_CommandMailbox_T *CommandMailbox = &Command->V2.CommandMailbox; - struct scatterlist *ScatterList = Command->V2.ScatterList; - - DAC960_V2_ClearCommand(Command); - - CommandMailbox->SCSI_10.CommandOpcode = DAC960_V2_SCSI_10; - CommandMailbox->SCSI_10.CommandControlBits.DataTransferControllerToHost = - (Command->DmaDirection == PCI_DMA_FROMDEVICE); - CommandMailbox->SCSI_10.DataTransferSize = - Command->BlockCount << DAC960_BlockSizeBits; - CommandMailbox->SCSI_10.RequestSenseBusAddress = Command->V2.RequestSenseDMA; - CommandMailbox->SCSI_10.PhysicalDevice = - Controller->V2.LogicalDriveToVirtualDevice[Command->LogicalDriveNumber]; - CommandMailbox->SCSI_10.RequestSenseSize = sizeof(DAC960_SCSI_RequestSense_T); - CommandMailbox->SCSI_10.CDBLength = 10; - CommandMailbox->SCSI_10.SCSI_CDB[0] = - (Command->DmaDirection == PCI_DMA_FROMDEVICE ? 0x28 : 0x2A); - CommandMailbox->SCSI_10.SCSI_CDB[2] = Command->BlockNumber >> 24; - CommandMailbox->SCSI_10.SCSI_CDB[3] = Command->BlockNumber >> 16; - CommandMailbox->SCSI_10.SCSI_CDB[4] = Command->BlockNumber >> 8; - CommandMailbox->SCSI_10.SCSI_CDB[5] = Command->BlockNumber; - CommandMailbox->SCSI_10.SCSI_CDB[7] = Command->BlockCount >> 8; - CommandMailbox->SCSI_10.SCSI_CDB[8] = Command->BlockCount; - - if (Command->SegmentCount == 1) - { - CommandMailbox->SCSI_10.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentDataPointer = - (DAC960_BusAddress64_T)sg_dma_address(ScatterList); - CommandMailbox->SCSI_10.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentByteCount = - CommandMailbox->SCSI_10.DataTransferSize; - } - else - { - DAC960_V2_ScatterGatherSegment_T *ScatterGatherList; - int i; - - if (Command->SegmentCount > 2) - { - ScatterGatherList = Command->V2.ScatterGatherList; - CommandMailbox->SCSI_10.CommandControlBits - .AdditionalScatterGatherListMemory = true; - CommandMailbox->SCSI_10.DataTransferMemoryAddress - .ExtendedScatterGather.ScatterGatherList0Length = Command->SegmentCount; - CommandMailbox->SCSI_10.DataTransferMemoryAddress - .ExtendedScatterGather.ScatterGatherList0Address = - Command->V2.ScatterGatherListDMA; - } - else - ScatterGatherList = CommandMailbox->SCSI_10.DataTransferMemoryAddress - .ScatterGatherSegments; - - for (i = 0; i < Command->SegmentCount; i++, ScatterList++, ScatterGatherList++) { - ScatterGatherList->SegmentDataPointer = - (DAC960_BusAddress64_T)sg_dma_address(ScatterList); - ScatterGatherList->SegmentByteCount = - (DAC960_ByteCount64_T)sg_dma_len(ScatterList); - } - } - DAC960_QueueCommand(Command); -} - - -static int DAC960_process_queue(DAC960_Controller_T *Controller, struct request_queue *req_q) -{ - struct request *Request; - DAC960_Command_T *Command; - - while(1) { - Request = blk_peek_request(req_q); - if (!Request) - return 1; - - Command = DAC960_AllocateCommand(Controller); - if (Command == NULL) - return 0; - - if (rq_data_dir(Request) == READ) { - Command->DmaDirection = PCI_DMA_FROMDEVICE; - Command->CommandType = DAC960_ReadCommand; - } else { - Command->DmaDirection = PCI_DMA_TODEVICE; - Command->CommandType = DAC960_WriteCommand; - } - Command->Completion = Request->end_io_data; - Command->LogicalDriveNumber = (long)Request->rq_disk->private_data; - Command->BlockNumber = blk_rq_pos(Request); - Command->BlockCount = blk_rq_sectors(Request); - Command->Request = Request; - blk_start_request(Request); - Command->SegmentCount = blk_rq_map_sg(req_q, - Command->Request, Command->cmd_sglist); - /* pci_map_sg MAY change the value of SegCount */ - Command->SegmentCount = pci_map_sg(Controller->PCIDevice, Command->cmd_sglist, - Command->SegmentCount, Command->DmaDirection); - - DAC960_QueueReadWriteCommand(Command); - } -} - -/* - DAC960_ProcessRequest attempts to remove one I/O Request from Controller's - I/O Request Queue and queues it to the Controller. WaitForCommand is true if - this function should wait for a Command to become available if necessary. - This function returns true if an I/O Request was queued and false otherwise. -*/ -static void DAC960_ProcessRequest(DAC960_Controller_T *controller) -{ - int i; - - if (!controller->ControllerInitialized) - return; - - /* Do this better later! */ - for (i = controller->req_q_index; i < DAC960_MaxLogicalDrives; i++) { - struct request_queue *req_q = controller->RequestQueue[i]; - - if (req_q == NULL) - continue; - - if (!DAC960_process_queue(controller, req_q)) { - controller->req_q_index = i; - return; - } - } - - if (controller->req_q_index == 0) - return; - - for (i = 0; i < controller->req_q_index; i++) { - struct request_queue *req_q = controller->RequestQueue[i]; - - if (req_q == NULL) - continue; - - if (!DAC960_process_queue(controller, req_q)) { - controller->req_q_index = i; - return; - } - } -} - - -/* - DAC960_queue_partial_rw extracts one bio from the request already - associated with argument command, and construct a new command block to retry I/O - only on that bio. Queue that command to the controller. - - This function re-uses a previously-allocated Command, - there is no failure mode from trying to allocate a command. -*/ - -static void DAC960_queue_partial_rw(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - struct request *Request = Command->Request; - struct request_queue *req_q = Controller->RequestQueue[Command->LogicalDriveNumber]; - - if (Command->DmaDirection == PCI_DMA_FROMDEVICE) - Command->CommandType = DAC960_ReadRetryCommand; - else - Command->CommandType = DAC960_WriteRetryCommand; - - /* - * We could be more efficient with these mapping requests - * and map only the portions that we need. But since this - * code should almost never be called, just go with a - * simple coding. - */ - (void)blk_rq_map_sg(req_q, Command->Request, Command->cmd_sglist); - - (void)pci_map_sg(Controller->PCIDevice, Command->cmd_sglist, 1, Command->DmaDirection); - /* - * Resubmitting the request sector at a time is really tedious. - * But, this should almost never happen. So, we're willing to pay - * this price so that in the end, as much of the transfer is completed - * successfully as possible. - */ - Command->SegmentCount = 1; - Command->BlockNumber = blk_rq_pos(Request); - Command->BlockCount = 1; - DAC960_QueueReadWriteCommand(Command); - return; -} - -/* - DAC960_RequestFunction is the I/O Request Function for DAC960 Controllers. -*/ - -static void DAC960_RequestFunction(struct request_queue *RequestQueue) -{ - DAC960_ProcessRequest(RequestQueue->queuedata); -} - -/* - DAC960_ProcessCompletedBuffer performs completion processing for an - individual Buffer. -*/ - -static inline bool DAC960_ProcessCompletedRequest(DAC960_Command_T *Command, - bool SuccessfulIO) -{ - struct request *Request = Command->Request; - blk_status_t Error = SuccessfulIO ? BLK_STS_OK : BLK_STS_IOERR; - - pci_unmap_sg(Command->Controller->PCIDevice, Command->cmd_sglist, - Command->SegmentCount, Command->DmaDirection); - - if (!__blk_end_request(Request, Error, Command->BlockCount << 9)) { - if (Command->Completion) { - complete(Command->Completion); - Command->Completion = NULL; - } - return true; - } - return false; -} - -/* - DAC960_V1_ReadWriteError prints an appropriate error message for Command - when an error occurs on a Read or Write operation. -*/ - -static void DAC960_V1_ReadWriteError(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - unsigned char *CommandName = "UNKNOWN"; - switch (Command->CommandType) - { - case DAC960_ReadCommand: - case DAC960_ReadRetryCommand: - CommandName = "READ"; - break; - case DAC960_WriteCommand: - case DAC960_WriteRetryCommand: - CommandName = "WRITE"; - break; - case DAC960_MonitoringCommand: - case DAC960_ImmediateCommand: - case DAC960_QueuedCommand: - break; - } - switch (Command->V1.CommandStatus) - { - case DAC960_V1_IrrecoverableDataError: - DAC960_Error("Irrecoverable Data Error on %s:\n", - Controller, CommandName); - break; - case DAC960_V1_LogicalDriveNonexistentOrOffline: - DAC960_Error("Logical Drive Nonexistent or Offline on %s:\n", - Controller, CommandName); - break; - case DAC960_V1_AccessBeyondEndOfLogicalDrive: - DAC960_Error("Attempt to Access Beyond End of Logical Drive " - "on %s:\n", Controller, CommandName); - break; - case DAC960_V1_BadDataEncountered: - DAC960_Error("Bad Data Encountered on %s:\n", Controller, CommandName); - break; - default: - DAC960_Error("Unexpected Error Status %04X on %s:\n", - Controller, Command->V1.CommandStatus, CommandName); - break; - } - DAC960_Error(" /dev/rd/c%dd%d: absolute blocks %u..%u\n", - Controller, Controller->ControllerNumber, - Command->LogicalDriveNumber, Command->BlockNumber, - Command->BlockNumber + Command->BlockCount - 1); -} - - -/* - DAC960_V1_ProcessCompletedCommand performs completion processing for Command - for DAC960 V1 Firmware Controllers. -*/ - -static void DAC960_V1_ProcessCompletedCommand(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - DAC960_CommandType_T CommandType = Command->CommandType; - DAC960_V1_CommandOpcode_T CommandOpcode = - Command->V1.CommandMailbox.Common.CommandOpcode; - DAC960_V1_CommandStatus_T CommandStatus = Command->V1.CommandStatus; - - if (CommandType == DAC960_ReadCommand || - CommandType == DAC960_WriteCommand) - { - -#ifdef FORCE_RETRY_DEBUG - CommandStatus = DAC960_V1_IrrecoverableDataError; -#endif - - if (CommandStatus == DAC960_V1_NormalCompletion) { - - if (!DAC960_ProcessCompletedRequest(Command, true)) - BUG(); - - } else if (CommandStatus == DAC960_V1_IrrecoverableDataError || - CommandStatus == DAC960_V1_BadDataEncountered) - { - /* - * break the command down into pieces and resubmit each - * piece, hoping that some of them will succeed. - */ - DAC960_queue_partial_rw(Command); - return; - } - else - { - if (CommandStatus != DAC960_V1_LogicalDriveNonexistentOrOffline) - DAC960_V1_ReadWriteError(Command); - - if (!DAC960_ProcessCompletedRequest(Command, false)) - BUG(); - } - } - else if (CommandType == DAC960_ReadRetryCommand || - CommandType == DAC960_WriteRetryCommand) - { - bool normal_completion; -#ifdef FORCE_RETRY_FAILURE_DEBUG - static int retry_count = 1; -#endif - /* - Perform completion processing for the portion that was - retried, and submit the next portion, if any. - */ - normal_completion = true; - if (CommandStatus != DAC960_V1_NormalCompletion) { - normal_completion = false; - if (CommandStatus != DAC960_V1_LogicalDriveNonexistentOrOffline) - DAC960_V1_ReadWriteError(Command); - } - -#ifdef FORCE_RETRY_FAILURE_DEBUG - if (!(++retry_count % 10000)) { - printk("V1 error retry failure test\n"); - normal_completion = false; - DAC960_V1_ReadWriteError(Command); - } -#endif - - if (!DAC960_ProcessCompletedRequest(Command, normal_completion)) { - DAC960_queue_partial_rw(Command); - return; - } - } - - else if (CommandType == DAC960_MonitoringCommand) - { - if (Controller->ShutdownMonitoringTimer) - return; - if (CommandOpcode == DAC960_V1_Enquiry) - { - DAC960_V1_Enquiry_T *OldEnquiry = &Controller->V1.Enquiry; - DAC960_V1_Enquiry_T *NewEnquiry = Controller->V1.NewEnquiry; - unsigned int OldCriticalLogicalDriveCount = - OldEnquiry->CriticalLogicalDriveCount; - unsigned int NewCriticalLogicalDriveCount = - NewEnquiry->CriticalLogicalDriveCount; - if (NewEnquiry->NumberOfLogicalDrives > Controller->LogicalDriveCount) - { - int LogicalDriveNumber = Controller->LogicalDriveCount - 1; - while (++LogicalDriveNumber < NewEnquiry->NumberOfLogicalDrives) - DAC960_Critical("Logical Drive %d (/dev/rd/c%dd%d) " - "Now Exists\n", Controller, - LogicalDriveNumber, - Controller->ControllerNumber, - LogicalDriveNumber); - Controller->LogicalDriveCount = NewEnquiry->NumberOfLogicalDrives; - DAC960_ComputeGenericDiskInfo(Controller); - } - if (NewEnquiry->NumberOfLogicalDrives < Controller->LogicalDriveCount) - { - int LogicalDriveNumber = NewEnquiry->NumberOfLogicalDrives - 1; - while (++LogicalDriveNumber < Controller->LogicalDriveCount) - DAC960_Critical("Logical Drive %d (/dev/rd/c%dd%d) " - "No Longer Exists\n", Controller, - LogicalDriveNumber, - Controller->ControllerNumber, - LogicalDriveNumber); - Controller->LogicalDriveCount = NewEnquiry->NumberOfLogicalDrives; - DAC960_ComputeGenericDiskInfo(Controller); - } - if (NewEnquiry->StatusFlags.DeferredWriteError != - OldEnquiry->StatusFlags.DeferredWriteError) - DAC960_Critical("Deferred Write Error Flag is now %s\n", Controller, - (NewEnquiry->StatusFlags.DeferredWriteError - ? "TRUE" : "FALSE")); - if ((NewCriticalLogicalDriveCount > 0 || - NewCriticalLogicalDriveCount != OldCriticalLogicalDriveCount) || - (NewEnquiry->OfflineLogicalDriveCount > 0 || - NewEnquiry->OfflineLogicalDriveCount != - OldEnquiry->OfflineLogicalDriveCount) || - (NewEnquiry->DeadDriveCount > 0 || - NewEnquiry->DeadDriveCount != - OldEnquiry->DeadDriveCount) || - (NewEnquiry->EventLogSequenceNumber != - OldEnquiry->EventLogSequenceNumber) || - Controller->MonitoringTimerCount == 0 || - time_after_eq(jiffies, Controller->SecondaryMonitoringTime - + DAC960_SecondaryMonitoringInterval)) - { - Controller->V1.NeedLogicalDriveInformation = true; - Controller->V1.NewEventLogSequenceNumber = - NewEnquiry->EventLogSequenceNumber; - Controller->V1.NeedErrorTableInformation = true; - Controller->V1.NeedDeviceStateInformation = true; - Controller->V1.StartDeviceStateScan = true; - Controller->V1.NeedBackgroundInitializationStatus = - Controller->V1.BackgroundInitializationStatusSupported; - Controller->SecondaryMonitoringTime = jiffies; - } - if (NewEnquiry->RebuildFlag == DAC960_V1_StandbyRebuildInProgress || - NewEnquiry->RebuildFlag - == DAC960_V1_BackgroundRebuildInProgress || - OldEnquiry->RebuildFlag == DAC960_V1_StandbyRebuildInProgress || - OldEnquiry->RebuildFlag == DAC960_V1_BackgroundRebuildInProgress) - { - Controller->V1.NeedRebuildProgress = true; - Controller->V1.RebuildProgressFirst = - (NewEnquiry->CriticalLogicalDriveCount < - OldEnquiry->CriticalLogicalDriveCount); - } - if (OldEnquiry->RebuildFlag == DAC960_V1_BackgroundCheckInProgress) - switch (NewEnquiry->RebuildFlag) - { - case DAC960_V1_NoStandbyRebuildOrCheckInProgress: - DAC960_Progress("Consistency Check Completed Successfully\n", - Controller); - break; - case DAC960_V1_StandbyRebuildInProgress: - case DAC960_V1_BackgroundRebuildInProgress: - break; - case DAC960_V1_BackgroundCheckInProgress: - Controller->V1.NeedConsistencyCheckProgress = true; - break; - case DAC960_V1_StandbyRebuildCompletedWithError: - DAC960_Progress("Consistency Check Completed with Error\n", - Controller); - break; - case DAC960_V1_BackgroundRebuildOrCheckFailed_DriveFailed: - DAC960_Progress("Consistency Check Failed - " - "Physical Device Failed\n", Controller); - break; - case DAC960_V1_BackgroundRebuildOrCheckFailed_LogicalDriveFailed: - DAC960_Progress("Consistency Check Failed - " - "Logical Drive Failed\n", Controller); - break; - case DAC960_V1_BackgroundRebuildOrCheckFailed_OtherCauses: - DAC960_Progress("Consistency Check Failed - Other Causes\n", - Controller); - break; - case DAC960_V1_BackgroundRebuildOrCheckSuccessfullyTerminated: - DAC960_Progress("Consistency Check Successfully Terminated\n", - Controller); - break; - } - else if (NewEnquiry->RebuildFlag - == DAC960_V1_BackgroundCheckInProgress) - Controller->V1.NeedConsistencyCheckProgress = true; - Controller->MonitoringAlertMode = - (NewEnquiry->CriticalLogicalDriveCount > 0 || - NewEnquiry->OfflineLogicalDriveCount > 0 || - NewEnquiry->DeadDriveCount > 0); - if (NewEnquiry->RebuildFlag > DAC960_V1_BackgroundCheckInProgress) - { - Controller->V1.PendingRebuildFlag = NewEnquiry->RebuildFlag; - Controller->V1.RebuildFlagPending = true; - } - memcpy(&Controller->V1.Enquiry, &Controller->V1.NewEnquiry, - sizeof(DAC960_V1_Enquiry_T)); - } - else if (CommandOpcode == DAC960_V1_PerformEventLogOperation) - { - static char - *DAC960_EventMessages[] = - { "killed because write recovery failed", - "killed because of SCSI bus reset failure", - "killed because of double check condition", - "killed because it was removed", - "killed because of gross error on SCSI chip", - "killed because of bad tag returned from drive", - "killed because of timeout on SCSI command", - "killed because of reset SCSI command issued from system", - "killed because busy or parity error count exceeded limit", - "killed because of 'kill drive' command from system", - "killed because of selection timeout", - "killed due to SCSI phase sequence error", - "killed due to unknown status" }; - DAC960_V1_EventLogEntry_T *EventLogEntry = - Controller->V1.EventLogEntry; - if (EventLogEntry->SequenceNumber == - Controller->V1.OldEventLogSequenceNumber) - { - unsigned char SenseKey = EventLogEntry->SenseKey; - unsigned char AdditionalSenseCode = - EventLogEntry->AdditionalSenseCode; - unsigned char AdditionalSenseCodeQualifier = - EventLogEntry->AdditionalSenseCodeQualifier; - if (SenseKey == DAC960_SenseKey_VendorSpecific && - AdditionalSenseCode == 0x80 && - AdditionalSenseCodeQualifier < - ARRAY_SIZE(DAC960_EventMessages)) - DAC960_Critical("Physical Device %d:%d %s\n", Controller, - EventLogEntry->Channel, - EventLogEntry->TargetID, - DAC960_EventMessages[ - AdditionalSenseCodeQualifier]); - else if (SenseKey == DAC960_SenseKey_UnitAttention && - AdditionalSenseCode == 0x29) - { - if (Controller->MonitoringTimerCount > 0) - Controller->V1.DeviceResetCount[EventLogEntry->Channel] - [EventLogEntry->TargetID]++; - } - else if (!(SenseKey == DAC960_SenseKey_NoSense || - (SenseKey == DAC960_SenseKey_NotReady && - AdditionalSenseCode == 0x04 && - (AdditionalSenseCodeQualifier == 0x01 || - AdditionalSenseCodeQualifier == 0x02)))) - { - DAC960_Critical("Physical Device %d:%d Error Log: " - "Sense Key = %X, ASC = %02X, ASCQ = %02X\n", - Controller, - EventLogEntry->Channel, - EventLogEntry->TargetID, - SenseKey, - AdditionalSenseCode, - AdditionalSenseCodeQualifier); - DAC960_Critical("Physical Device %d:%d Error Log: " - "Information = %02X%02X%02X%02X " - "%02X%02X%02X%02X\n", - Controller, - EventLogEntry->Channel, - EventLogEntry->TargetID, - EventLogEntry->Information[0], - EventLogEntry->Information[1], - EventLogEntry->Information[2], - EventLogEntry->Information[3], - EventLogEntry->CommandSpecificInformation[0], - EventLogEntry->CommandSpecificInformation[1], - EventLogEntry->CommandSpecificInformation[2], - EventLogEntry->CommandSpecificInformation[3]); - } - } - Controller->V1.OldEventLogSequenceNumber++; - } - else if (CommandOpcode == DAC960_V1_GetErrorTable) - { - DAC960_V1_ErrorTable_T *OldErrorTable = &Controller->V1.ErrorTable; - DAC960_V1_ErrorTable_T *NewErrorTable = Controller->V1.NewErrorTable; - int Channel, TargetID; - for (Channel = 0; Channel < Controller->Channels; Channel++) - for (TargetID = 0; TargetID < Controller->Targets; TargetID++) - { - DAC960_V1_ErrorTableEntry_T *NewErrorEntry = - &NewErrorTable->ErrorTableEntries[Channel][TargetID]; - DAC960_V1_ErrorTableEntry_T *OldErrorEntry = - &OldErrorTable->ErrorTableEntries[Channel][TargetID]; - if ((NewErrorEntry->ParityErrorCount != - OldErrorEntry->ParityErrorCount) || - (NewErrorEntry->SoftErrorCount != - OldErrorEntry->SoftErrorCount) || - (NewErrorEntry->HardErrorCount != - OldErrorEntry->HardErrorCount) || - (NewErrorEntry->MiscErrorCount != - OldErrorEntry->MiscErrorCount)) - DAC960_Critical("Physical Device %d:%d Errors: " - "Parity = %d, Soft = %d, " - "Hard = %d, Misc = %d\n", - Controller, Channel, TargetID, - NewErrorEntry->ParityErrorCount, - NewErrorEntry->SoftErrorCount, - NewErrorEntry->HardErrorCount, - NewErrorEntry->MiscErrorCount); - } - memcpy(&Controller->V1.ErrorTable, Controller->V1.NewErrorTable, - sizeof(DAC960_V1_ErrorTable_T)); - } - else if (CommandOpcode == DAC960_V1_GetDeviceState) - { - DAC960_V1_DeviceState_T *OldDeviceState = - &Controller->V1.DeviceState[Controller->V1.DeviceStateChannel] - [Controller->V1.DeviceStateTargetID]; - DAC960_V1_DeviceState_T *NewDeviceState = - Controller->V1.NewDeviceState; - if (NewDeviceState->DeviceState != OldDeviceState->DeviceState) - DAC960_Critical("Physical Device %d:%d is now %s\n", Controller, - Controller->V1.DeviceStateChannel, - Controller->V1.DeviceStateTargetID, - (NewDeviceState->DeviceState - == DAC960_V1_Device_Dead - ? "DEAD" - : NewDeviceState->DeviceState - == DAC960_V1_Device_WriteOnly - ? "WRITE-ONLY" - : NewDeviceState->DeviceState - == DAC960_V1_Device_Online - ? "ONLINE" : "STANDBY")); - if (OldDeviceState->DeviceState == DAC960_V1_Device_Dead && - NewDeviceState->DeviceState != DAC960_V1_Device_Dead) - { - Controller->V1.NeedDeviceInquiryInformation = true; - Controller->V1.NeedDeviceSerialNumberInformation = true; - Controller->V1.DeviceResetCount - [Controller->V1.DeviceStateChannel] - [Controller->V1.DeviceStateTargetID] = 0; - } - memcpy(OldDeviceState, NewDeviceState, - sizeof(DAC960_V1_DeviceState_T)); - } - else if (CommandOpcode == DAC960_V1_GetLogicalDriveInformation) - { - int LogicalDriveNumber; - for (LogicalDriveNumber = 0; - LogicalDriveNumber < Controller->LogicalDriveCount; - LogicalDriveNumber++) - { - DAC960_V1_LogicalDriveInformation_T *OldLogicalDriveInformation = - &Controller->V1.LogicalDriveInformation[LogicalDriveNumber]; - DAC960_V1_LogicalDriveInformation_T *NewLogicalDriveInformation = - &(*Controller->V1.NewLogicalDriveInformation)[LogicalDriveNumber]; - if (NewLogicalDriveInformation->LogicalDriveState != - OldLogicalDriveInformation->LogicalDriveState) - DAC960_Critical("Logical Drive %d (/dev/rd/c%dd%d) " - "is now %s\n", Controller, - LogicalDriveNumber, - Controller->ControllerNumber, - LogicalDriveNumber, - (NewLogicalDriveInformation->LogicalDriveState - == DAC960_V1_LogicalDrive_Online - ? "ONLINE" - : NewLogicalDriveInformation->LogicalDriveState - == DAC960_V1_LogicalDrive_Critical - ? "CRITICAL" : "OFFLINE")); - if (NewLogicalDriveInformation->WriteBack != - OldLogicalDriveInformation->WriteBack) - DAC960_Critical("Logical Drive %d (/dev/rd/c%dd%d) " - "is now %s\n", Controller, - LogicalDriveNumber, - Controller->ControllerNumber, - LogicalDriveNumber, - (NewLogicalDriveInformation->WriteBack - ? "WRITE BACK" : "WRITE THRU")); - } - memcpy(&Controller->V1.LogicalDriveInformation, - Controller->V1.NewLogicalDriveInformation, - sizeof(DAC960_V1_LogicalDriveInformationArray_T)); - } - else if (CommandOpcode == DAC960_V1_GetRebuildProgress) - { - unsigned int LogicalDriveNumber = - Controller->V1.RebuildProgress->LogicalDriveNumber; - unsigned int LogicalDriveSize = - Controller->V1.RebuildProgress->LogicalDriveSize; - unsigned int BlocksCompleted = - LogicalDriveSize - Controller->V1.RebuildProgress->RemainingBlocks; - if (CommandStatus == DAC960_V1_NoRebuildOrCheckInProgress && - Controller->V1.LastRebuildStatus == DAC960_V1_NormalCompletion) - CommandStatus = DAC960_V1_RebuildSuccessful; - switch (CommandStatus) - { - case DAC960_V1_NormalCompletion: - Controller->EphemeralProgressMessage = true; - DAC960_Progress("Rebuild in Progress: " - "Logical Drive %d (/dev/rd/c%dd%d) " - "%d%% completed\n", - Controller, LogicalDriveNumber, - Controller->ControllerNumber, - LogicalDriveNumber, - (100 * (BlocksCompleted >> 7)) - / (LogicalDriveSize >> 7)); - Controller->EphemeralProgressMessage = false; - break; - case DAC960_V1_RebuildFailed_LogicalDriveFailure: - DAC960_Progress("Rebuild Failed due to " - "Logical Drive Failure\n", Controller); - break; - case DAC960_V1_RebuildFailed_BadBlocksOnOther: - DAC960_Progress("Rebuild Failed due to " - "Bad Blocks on Other Drives\n", Controller); - break; - case DAC960_V1_RebuildFailed_NewDriveFailed: - DAC960_Progress("Rebuild Failed due to " - "Failure of Drive Being Rebuilt\n", Controller); - break; - case DAC960_V1_NoRebuildOrCheckInProgress: - break; - case DAC960_V1_RebuildSuccessful: - DAC960_Progress("Rebuild Completed Successfully\n", Controller); - break; - case DAC960_V1_RebuildSuccessfullyTerminated: - DAC960_Progress("Rebuild Successfully Terminated\n", Controller); - break; - } - Controller->V1.LastRebuildStatus = CommandStatus; - if (CommandType != DAC960_MonitoringCommand && - Controller->V1.RebuildStatusPending) - { - Command->V1.CommandStatus = Controller->V1.PendingRebuildStatus; - Controller->V1.RebuildStatusPending = false; - } - else if (CommandType == DAC960_MonitoringCommand && - CommandStatus != DAC960_V1_NormalCompletion && - CommandStatus != DAC960_V1_NoRebuildOrCheckInProgress) - { - Controller->V1.PendingRebuildStatus = CommandStatus; - Controller->V1.RebuildStatusPending = true; - } - } - else if (CommandOpcode == DAC960_V1_RebuildStat) - { - unsigned int LogicalDriveNumber = - Controller->V1.RebuildProgress->LogicalDriveNumber; - unsigned int LogicalDriveSize = - Controller->V1.RebuildProgress->LogicalDriveSize; - unsigned int BlocksCompleted = - LogicalDriveSize - Controller->V1.RebuildProgress->RemainingBlocks; - if (CommandStatus == DAC960_V1_NormalCompletion) - { - Controller->EphemeralProgressMessage = true; - DAC960_Progress("Consistency Check in Progress: " - "Logical Drive %d (/dev/rd/c%dd%d) " - "%d%% completed\n", - Controller, LogicalDriveNumber, - Controller->ControllerNumber, - LogicalDriveNumber, - (100 * (BlocksCompleted >> 7)) - / (LogicalDriveSize >> 7)); - Controller->EphemeralProgressMessage = false; - } - } - else if (CommandOpcode == DAC960_V1_BackgroundInitializationControl) - { - unsigned int LogicalDriveNumber = - Controller->V1.BackgroundInitializationStatus->LogicalDriveNumber; - unsigned int LogicalDriveSize = - Controller->V1.BackgroundInitializationStatus->LogicalDriveSize; - unsigned int BlocksCompleted = - Controller->V1.BackgroundInitializationStatus->BlocksCompleted; - switch (CommandStatus) - { - case DAC960_V1_NormalCompletion: - switch (Controller->V1.BackgroundInitializationStatus->Status) - { - case DAC960_V1_BackgroundInitializationInvalid: - break; - case DAC960_V1_BackgroundInitializationStarted: - DAC960_Progress("Background Initialization Started\n", - Controller); - break; - case DAC960_V1_BackgroundInitializationInProgress: - if (BlocksCompleted == - Controller->V1.LastBackgroundInitializationStatus. - BlocksCompleted && - LogicalDriveNumber == - Controller->V1.LastBackgroundInitializationStatus. - LogicalDriveNumber) - break; - Controller->EphemeralProgressMessage = true; - DAC960_Progress("Background Initialization in Progress: " - "Logical Drive %d (/dev/rd/c%dd%d) " - "%d%% completed\n", - Controller, LogicalDriveNumber, - Controller->ControllerNumber, - LogicalDriveNumber, - (100 * (BlocksCompleted >> 7)) - / (LogicalDriveSize >> 7)); - Controller->EphemeralProgressMessage = false; - break; - case DAC960_V1_BackgroundInitializationSuspended: - DAC960_Progress("Background Initialization Suspended\n", - Controller); - break; - case DAC960_V1_BackgroundInitializationCancelled: - DAC960_Progress("Background Initialization Cancelled\n", - Controller); - break; - } - memcpy(&Controller->V1.LastBackgroundInitializationStatus, - Controller->V1.BackgroundInitializationStatus, - sizeof(DAC960_V1_BackgroundInitializationStatus_T)); - break; - case DAC960_V1_BackgroundInitSuccessful: - if (Controller->V1.BackgroundInitializationStatus->Status == - DAC960_V1_BackgroundInitializationInProgress) - DAC960_Progress("Background Initialization " - "Completed Successfully\n", Controller); - Controller->V1.BackgroundInitializationStatus->Status = - DAC960_V1_BackgroundInitializationInvalid; - break; - case DAC960_V1_BackgroundInitAborted: - if (Controller->V1.BackgroundInitializationStatus->Status == - DAC960_V1_BackgroundInitializationInProgress) - DAC960_Progress("Background Initialization Aborted\n", - Controller); - Controller->V1.BackgroundInitializationStatus->Status = - DAC960_V1_BackgroundInitializationInvalid; - break; - case DAC960_V1_NoBackgroundInitInProgress: - break; - } - } - else if (CommandOpcode == DAC960_V1_DCDB) - { - /* - This is a bit ugly. - - The InquiryStandardData and - the InquiryUntitSerialNumber information - retrieval operations BOTH use the DAC960_V1_DCDB - commands. the test above can't distinguish between - these two cases. - - Instead, we rely on the order of code later in this - function to ensure that DeviceInquiryInformation commands - are submitted before DeviceSerialNumber commands. - */ - if (Controller->V1.NeedDeviceInquiryInformation) - { - DAC960_SCSI_Inquiry_T *InquiryStandardData = - &Controller->V1.InquiryStandardData - [Controller->V1.DeviceStateChannel] - [Controller->V1.DeviceStateTargetID]; - if (CommandStatus != DAC960_V1_NormalCompletion) - { - memset(InquiryStandardData, 0, - sizeof(DAC960_SCSI_Inquiry_T)); - InquiryStandardData->PeripheralDeviceType = 0x1F; - } - else - memcpy(InquiryStandardData, - Controller->V1.NewInquiryStandardData, - sizeof(DAC960_SCSI_Inquiry_T)); - Controller->V1.NeedDeviceInquiryInformation = false; - } - else if (Controller->V1.NeedDeviceSerialNumberInformation) - { - DAC960_SCSI_Inquiry_UnitSerialNumber_T *InquiryUnitSerialNumber = - &Controller->V1.InquiryUnitSerialNumber - [Controller->V1.DeviceStateChannel] - [Controller->V1.DeviceStateTargetID]; - if (CommandStatus != DAC960_V1_NormalCompletion) - { - memset(InquiryUnitSerialNumber, 0, - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T)); - InquiryUnitSerialNumber->PeripheralDeviceType = 0x1F; - } - else - memcpy(InquiryUnitSerialNumber, - Controller->V1.NewInquiryUnitSerialNumber, - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T)); - Controller->V1.NeedDeviceSerialNumberInformation = false; - } - } - /* - Begin submitting new monitoring commands. - */ - if (Controller->V1.NewEventLogSequenceNumber - - Controller->V1.OldEventLogSequenceNumber > 0) - { - Command->V1.CommandMailbox.Type3E.CommandOpcode = - DAC960_V1_PerformEventLogOperation; - Command->V1.CommandMailbox.Type3E.OperationType = - DAC960_V1_GetEventLogEntry; - Command->V1.CommandMailbox.Type3E.OperationQualifier = 1; - Command->V1.CommandMailbox.Type3E.SequenceNumber = - Controller->V1.OldEventLogSequenceNumber; - Command->V1.CommandMailbox.Type3E.BusAddress = - Controller->V1.EventLogEntryDMA; - DAC960_QueueCommand(Command); - return; - } - if (Controller->V1.NeedErrorTableInformation) - { - Controller->V1.NeedErrorTableInformation = false; - Command->V1.CommandMailbox.Type3.CommandOpcode = - DAC960_V1_GetErrorTable; - Command->V1.CommandMailbox.Type3.BusAddress = - Controller->V1.NewErrorTableDMA; - DAC960_QueueCommand(Command); - return; - } - if (Controller->V1.NeedRebuildProgress && - Controller->V1.RebuildProgressFirst) - { - Controller->V1.NeedRebuildProgress = false; - Command->V1.CommandMailbox.Type3.CommandOpcode = - DAC960_V1_GetRebuildProgress; - Command->V1.CommandMailbox.Type3.BusAddress = - Controller->V1.RebuildProgressDMA; - DAC960_QueueCommand(Command); - return; - } - if (Controller->V1.NeedDeviceStateInformation) - { - if (Controller->V1.NeedDeviceInquiryInformation) - { - DAC960_V1_DCDB_T *DCDB = Controller->V1.MonitoringDCDB; - dma_addr_t DCDB_DMA = Controller->V1.MonitoringDCDB_DMA; - - dma_addr_t NewInquiryStandardDataDMA = - Controller->V1.NewInquiryStandardDataDMA; - - Command->V1.CommandMailbox.Type3.CommandOpcode = DAC960_V1_DCDB; - Command->V1.CommandMailbox.Type3.BusAddress = DCDB_DMA; - DCDB->Channel = Controller->V1.DeviceStateChannel; - DCDB->TargetID = Controller->V1.DeviceStateTargetID; - DCDB->Direction = DAC960_V1_DCDB_DataTransferDeviceToSystem; - DCDB->EarlyStatus = false; - DCDB->Timeout = DAC960_V1_DCDB_Timeout_10_seconds; - DCDB->NoAutomaticRequestSense = false; - DCDB->DisconnectPermitted = true; - DCDB->TransferLength = sizeof(DAC960_SCSI_Inquiry_T); - DCDB->BusAddress = NewInquiryStandardDataDMA; - DCDB->CDBLength = 6; - DCDB->TransferLengthHigh4 = 0; - DCDB->SenseLength = sizeof(DCDB->SenseData); - DCDB->CDB[0] = 0x12; /* INQUIRY */ - DCDB->CDB[1] = 0; /* EVPD = 0 */ - DCDB->CDB[2] = 0; /* Page Code */ - DCDB->CDB[3] = 0; /* Reserved */ - DCDB->CDB[4] = sizeof(DAC960_SCSI_Inquiry_T); - DCDB->CDB[5] = 0; /* Control */ - DAC960_QueueCommand(Command); - return; - } - if (Controller->V1.NeedDeviceSerialNumberInformation) - { - DAC960_V1_DCDB_T *DCDB = Controller->V1.MonitoringDCDB; - dma_addr_t DCDB_DMA = Controller->V1.MonitoringDCDB_DMA; - dma_addr_t NewInquiryUnitSerialNumberDMA = - Controller->V1.NewInquiryUnitSerialNumberDMA; - - Command->V1.CommandMailbox.Type3.CommandOpcode = DAC960_V1_DCDB; - Command->V1.CommandMailbox.Type3.BusAddress = DCDB_DMA; - DCDB->Channel = Controller->V1.DeviceStateChannel; - DCDB->TargetID = Controller->V1.DeviceStateTargetID; - DCDB->Direction = DAC960_V1_DCDB_DataTransferDeviceToSystem; - DCDB->EarlyStatus = false; - DCDB->Timeout = DAC960_V1_DCDB_Timeout_10_seconds; - DCDB->NoAutomaticRequestSense = false; - DCDB->DisconnectPermitted = true; - DCDB->TransferLength = - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T); - DCDB->BusAddress = NewInquiryUnitSerialNumberDMA; - DCDB->CDBLength = 6; - DCDB->TransferLengthHigh4 = 0; - DCDB->SenseLength = sizeof(DCDB->SenseData); - DCDB->CDB[0] = 0x12; /* INQUIRY */ - DCDB->CDB[1] = 1; /* EVPD = 1 */ - DCDB->CDB[2] = 0x80; /* Page Code */ - DCDB->CDB[3] = 0; /* Reserved */ - DCDB->CDB[4] = sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T); - DCDB->CDB[5] = 0; /* Control */ - DAC960_QueueCommand(Command); - return; - } - if (Controller->V1.StartDeviceStateScan) - { - Controller->V1.DeviceStateChannel = 0; - Controller->V1.DeviceStateTargetID = 0; - Controller->V1.StartDeviceStateScan = false; - } - else if (++Controller->V1.DeviceStateTargetID == Controller->Targets) - { - Controller->V1.DeviceStateChannel++; - Controller->V1.DeviceStateTargetID = 0; - } - if (Controller->V1.DeviceStateChannel < Controller->Channels) - { - Controller->V1.NewDeviceState->DeviceState = - DAC960_V1_Device_Dead; - Command->V1.CommandMailbox.Type3D.CommandOpcode = - DAC960_V1_GetDeviceState; - Command->V1.CommandMailbox.Type3D.Channel = - Controller->V1.DeviceStateChannel; - Command->V1.CommandMailbox.Type3D.TargetID = - Controller->V1.DeviceStateTargetID; - Command->V1.CommandMailbox.Type3D.BusAddress = - Controller->V1.NewDeviceStateDMA; - DAC960_QueueCommand(Command); - return; - } - Controller->V1.NeedDeviceStateInformation = false; - } - if (Controller->V1.NeedLogicalDriveInformation) - { - Controller->V1.NeedLogicalDriveInformation = false; - Command->V1.CommandMailbox.Type3.CommandOpcode = - DAC960_V1_GetLogicalDriveInformation; - Command->V1.CommandMailbox.Type3.BusAddress = - Controller->V1.NewLogicalDriveInformationDMA; - DAC960_QueueCommand(Command); - return; - } - if (Controller->V1.NeedRebuildProgress) - { - Controller->V1.NeedRebuildProgress = false; - Command->V1.CommandMailbox.Type3.CommandOpcode = - DAC960_V1_GetRebuildProgress; - Command->V1.CommandMailbox.Type3.BusAddress = - Controller->V1.RebuildProgressDMA; - DAC960_QueueCommand(Command); - return; - } - if (Controller->V1.NeedConsistencyCheckProgress) - { - Controller->V1.NeedConsistencyCheckProgress = false; - Command->V1.CommandMailbox.Type3.CommandOpcode = - DAC960_V1_RebuildStat; - Command->V1.CommandMailbox.Type3.BusAddress = - Controller->V1.RebuildProgressDMA; - DAC960_QueueCommand(Command); - return; - } - if (Controller->V1.NeedBackgroundInitializationStatus) - { - Controller->V1.NeedBackgroundInitializationStatus = false; - Command->V1.CommandMailbox.Type3B.CommandOpcode = - DAC960_V1_BackgroundInitializationControl; - Command->V1.CommandMailbox.Type3B.CommandOpcode2 = 0x20; - Command->V1.CommandMailbox.Type3B.BusAddress = - Controller->V1.BackgroundInitializationStatusDMA; - DAC960_QueueCommand(Command); - return; - } - Controller->MonitoringTimerCount++; - Controller->MonitoringTimer.expires = - jiffies + DAC960_MonitoringTimerInterval; - add_timer(&Controller->MonitoringTimer); - } - if (CommandType == DAC960_ImmediateCommand) - { - complete(Command->Completion); - Command->Completion = NULL; - return; - } - if (CommandType == DAC960_QueuedCommand) - { - DAC960_V1_KernelCommand_T *KernelCommand = Command->V1.KernelCommand; - KernelCommand->CommandStatus = Command->V1.CommandStatus; - Command->V1.KernelCommand = NULL; - if (CommandOpcode == DAC960_V1_DCDB) - Controller->V1.DirectCommandActive[KernelCommand->DCDB->Channel] - [KernelCommand->DCDB->TargetID] = - false; - DAC960_DeallocateCommand(Command); - KernelCommand->CompletionFunction(KernelCommand); - return; - } - /* - Queue a Status Monitoring Command to the Controller using the just - completed Command if one was deferred previously due to lack of a - free Command when the Monitoring Timer Function was called. - */ - if (Controller->MonitoringCommandDeferred) - { - Controller->MonitoringCommandDeferred = false; - DAC960_V1_QueueMonitoringCommand(Command); - return; - } - /* - Deallocate the Command. - */ - DAC960_DeallocateCommand(Command); - /* - Wake up any processes waiting on a free Command. - */ - wake_up(&Controller->CommandWaitQueue); -} - - -/* - DAC960_V2_ReadWriteError prints an appropriate error message for Command - when an error occurs on a Read or Write operation. -*/ - -static void DAC960_V2_ReadWriteError(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - static const unsigned char *SenseErrors[] = { - "NO SENSE", "RECOVERED ERROR", - "NOT READY", "MEDIUM ERROR", - "HARDWARE ERROR", "ILLEGAL REQUEST", - "UNIT ATTENTION", "DATA PROTECT", - "BLANK CHECK", "VENDOR-SPECIFIC", - "COPY ABORTED", "ABORTED COMMAND", - "EQUAL", "VOLUME OVERFLOW", - "MISCOMPARE", "RESERVED" - }; - unsigned char *CommandName = "UNKNOWN"; - switch (Command->CommandType) - { - case DAC960_ReadCommand: - case DAC960_ReadRetryCommand: - CommandName = "READ"; - break; - case DAC960_WriteCommand: - case DAC960_WriteRetryCommand: - CommandName = "WRITE"; - break; - case DAC960_MonitoringCommand: - case DAC960_ImmediateCommand: - case DAC960_QueuedCommand: - break; - } - DAC960_Error("Error Condition %s on %s:\n", Controller, - SenseErrors[Command->V2.RequestSense->SenseKey], CommandName); - DAC960_Error(" /dev/rd/c%dd%d: absolute blocks %u..%u\n", - Controller, Controller->ControllerNumber, - Command->LogicalDriveNumber, Command->BlockNumber, - Command->BlockNumber + Command->BlockCount - 1); -} - - -/* - DAC960_V2_ReportEvent prints an appropriate message when a Controller Event - occurs. -*/ - -static void DAC960_V2_ReportEvent(DAC960_Controller_T *Controller, - DAC960_V2_Event_T *Event) -{ - DAC960_SCSI_RequestSense_T *RequestSense = - (DAC960_SCSI_RequestSense_T *) &Event->RequestSenseData; - unsigned char MessageBuffer[DAC960_LineBufferSize]; - static struct { int EventCode; unsigned char *EventMessage; } EventList[] = - { /* Physical Device Events (0x0000 - 0x007F) */ - { 0x0001, "P Online" }, - { 0x0002, "P Standby" }, - { 0x0005, "P Automatic Rebuild Started" }, - { 0x0006, "P Manual Rebuild Started" }, - { 0x0007, "P Rebuild Completed" }, - { 0x0008, "P Rebuild Cancelled" }, - { 0x0009, "P Rebuild Failed for Unknown Reasons" }, - { 0x000A, "P Rebuild Failed due to New Physical Device" }, - { 0x000B, "P Rebuild Failed due to Logical Drive Failure" }, - { 0x000C, "S Offline" }, - { 0x000D, "P Found" }, - { 0x000E, "P Removed" }, - { 0x000F, "P Unconfigured" }, - { 0x0010, "P Expand Capacity Started" }, - { 0x0011, "P Expand Capacity Completed" }, - { 0x0012, "P Expand Capacity Failed" }, - { 0x0013, "P Command Timed Out" }, - { 0x0014, "P Command Aborted" }, - { 0x0015, "P Command Retried" }, - { 0x0016, "P Parity Error" }, - { 0x0017, "P Soft Error" }, - { 0x0018, "P Miscellaneous Error" }, - { 0x0019, "P Reset" }, - { 0x001A, "P Active Spare Found" }, - { 0x001B, "P Warm Spare Found" }, - { 0x001C, "S Sense Data Received" }, - { 0x001D, "P Initialization Started" }, - { 0x001E, "P Initialization Completed" }, - { 0x001F, "P Initialization Failed" }, - { 0x0020, "P Initialization Cancelled" }, - { 0x0021, "P Failed because Write Recovery Failed" }, - { 0x0022, "P Failed because SCSI Bus Reset Failed" }, - { 0x0023, "P Failed because of Double Check Condition" }, - { 0x0024, "P Failed because Device Cannot Be Accessed" }, - { 0x0025, "P Failed because of Gross Error on SCSI Processor" }, - { 0x0026, "P Failed because of Bad Tag from Device" }, - { 0x0027, "P Failed because of Command Timeout" }, - { 0x0028, "P Failed because of System Reset" }, - { 0x0029, "P Failed because of Busy Status or Parity Error" }, - { 0x002A, "P Failed because Host Set Device to Failed State" }, - { 0x002B, "P Failed because of Selection Timeout" }, - { 0x002C, "P Failed because of SCSI Bus Phase Error" }, - { 0x002D, "P Failed because Device Returned Unknown Status" }, - { 0x002E, "P Failed because Device Not Ready" }, - { 0x002F, "P Failed because Device Not Found at Startup" }, - { 0x0030, "P Failed because COD Write Operation Failed" }, - { 0x0031, "P Failed because BDT Write Operation Failed" }, - { 0x0039, "P Missing at Startup" }, - { 0x003A, "P Start Rebuild Failed due to Physical Drive Too Small" }, - { 0x003C, "P Temporarily Offline Device Automatically Made Online" }, - { 0x003D, "P Standby Rebuild Started" }, - /* Logical Device Events (0x0080 - 0x00FF) */ - { 0x0080, "M Consistency Check Started" }, - { 0x0081, "M Consistency Check Completed" }, - { 0x0082, "M Consistency Check Cancelled" }, - { 0x0083, "M Consistency Check Completed With Errors" }, - { 0x0084, "M Consistency Check Failed due to Logical Drive Failure" }, - { 0x0085, "M Consistency Check Failed due to Physical Device Failure" }, - { 0x0086, "L Offline" }, - { 0x0087, "L Critical" }, - { 0x0088, "L Online" }, - { 0x0089, "M Automatic Rebuild Started" }, - { 0x008A, "M Manual Rebuild Started" }, - { 0x008B, "M Rebuild Completed" }, - { 0x008C, "M Rebuild Cancelled" }, - { 0x008D, "M Rebuild Failed for Unknown Reasons" }, - { 0x008E, "M Rebuild Failed due to New Physical Device" }, - { 0x008F, "M Rebuild Failed due to Logical Drive Failure" }, - { 0x0090, "M Initialization Started" }, - { 0x0091, "M Initialization Completed" }, - { 0x0092, "M Initialization Cancelled" }, - { 0x0093, "M Initialization Failed" }, - { 0x0094, "L Found" }, - { 0x0095, "L Deleted" }, - { 0x0096, "M Expand Capacity Started" }, - { 0x0097, "M Expand Capacity Completed" }, - { 0x0098, "M Expand Capacity Failed" }, - { 0x0099, "L Bad Block Found" }, - { 0x009A, "L Size Changed" }, - { 0x009B, "L Type Changed" }, - { 0x009C, "L Bad Data Block Found" }, - { 0x009E, "L Read of Data Block in BDT" }, - { 0x009F, "L Write Back Data for Disk Block Lost" }, - { 0x00A0, "L Temporarily Offline RAID-5/3 Drive Made Online" }, - { 0x00A1, "L Temporarily Offline RAID-6/1/0/7 Drive Made Online" }, - { 0x00A2, "L Standby Rebuild Started" }, - /* Fault Management Events (0x0100 - 0x017F) */ - { 0x0140, "E Fan %d Failed" }, - { 0x0141, "E Fan %d OK" }, - { 0x0142, "E Fan %d Not Present" }, - { 0x0143, "E Power Supply %d Failed" }, - { 0x0144, "E Power Supply %d OK" }, - { 0x0145, "E Power Supply %d Not Present" }, - { 0x0146, "E Temperature Sensor %d Temperature Exceeds Safe Limit" }, - { 0x0147, "E Temperature Sensor %d Temperature Exceeds Working Limit" }, - { 0x0148, "E Temperature Sensor %d Temperature Normal" }, - { 0x0149, "E Temperature Sensor %d Not Present" }, - { 0x014A, "E Enclosure Management Unit %d Access Critical" }, - { 0x014B, "E Enclosure Management Unit %d Access OK" }, - { 0x014C, "E Enclosure Management Unit %d Access Offline" }, - /* Controller Events (0x0180 - 0x01FF) */ - { 0x0181, "C Cache Write Back Error" }, - { 0x0188, "C Battery Backup Unit Found" }, - { 0x0189, "C Battery Backup Unit Charge Level Low" }, - { 0x018A, "C Battery Backup Unit Charge Level OK" }, - { 0x0193, "C Installation Aborted" }, - { 0x0195, "C Battery Backup Unit Physically Removed" }, - { 0x0196, "C Memory Error During Warm Boot" }, - { 0x019E, "C Memory Soft ECC Error Corrected" }, - { 0x019F, "C Memory Hard ECC Error Corrected" }, - { 0x01A2, "C Battery Backup Unit Failed" }, - { 0x01AB, "C Mirror Race Recovery Failed" }, - { 0x01AC, "C Mirror Race on Critical Drive" }, - /* Controller Internal Processor Events */ - { 0x0380, "C Internal Controller Hung" }, - { 0x0381, "C Internal Controller Firmware Breakpoint" }, - { 0x0390, "C Internal Controller i960 Processor Specific Error" }, - { 0x03A0, "C Internal Controller StrongARM Processor Specific Error" }, - { 0, "" } }; - int EventListIndex = 0, EventCode; - unsigned char EventType, *EventMessage; - if (Event->EventCode == 0x1C && - RequestSense->SenseKey == DAC960_SenseKey_VendorSpecific && - (RequestSense->AdditionalSenseCode == 0x80 || - RequestSense->AdditionalSenseCode == 0x81)) - Event->EventCode = ((RequestSense->AdditionalSenseCode - 0x80) << 8) | - RequestSense->AdditionalSenseCodeQualifier; - while (true) - { - EventCode = EventList[EventListIndex].EventCode; - if (EventCode == Event->EventCode || EventCode == 0) break; - EventListIndex++; - } - EventType = EventList[EventListIndex].EventMessage[0]; - EventMessage = &EventList[EventListIndex].EventMessage[2]; - if (EventCode == 0) - { - DAC960_Critical("Unknown Controller Event Code %04X\n", - Controller, Event->EventCode); - return; - } - switch (EventType) - { - case 'P': - DAC960_Critical("Physical Device %d:%d %s\n", Controller, - Event->Channel, Event->TargetID, EventMessage); - break; - case 'L': - DAC960_Critical("Logical Drive %d (/dev/rd/c%dd%d) %s\n", Controller, - Event->LogicalUnit, Controller->ControllerNumber, - Event->LogicalUnit, EventMessage); - break; - case 'M': - DAC960_Progress("Logical Drive %d (/dev/rd/c%dd%d) %s\n", Controller, - Event->LogicalUnit, Controller->ControllerNumber, - Event->LogicalUnit, EventMessage); - break; - case 'S': - if (RequestSense->SenseKey == DAC960_SenseKey_NoSense || - (RequestSense->SenseKey == DAC960_SenseKey_NotReady && - RequestSense->AdditionalSenseCode == 0x04 && - (RequestSense->AdditionalSenseCodeQualifier == 0x01 || - RequestSense->AdditionalSenseCodeQualifier == 0x02))) - break; - DAC960_Critical("Physical Device %d:%d %s\n", Controller, - Event->Channel, Event->TargetID, EventMessage); - DAC960_Critical("Physical Device %d:%d Request Sense: " - "Sense Key = %X, ASC = %02X, ASCQ = %02X\n", - Controller, - Event->Channel, - Event->TargetID, - RequestSense->SenseKey, - RequestSense->AdditionalSenseCode, - RequestSense->AdditionalSenseCodeQualifier); - DAC960_Critical("Physical Device %d:%d Request Sense: " - "Information = %02X%02X%02X%02X " - "%02X%02X%02X%02X\n", - Controller, - Event->Channel, - Event->TargetID, - RequestSense->Information[0], - RequestSense->Information[1], - RequestSense->Information[2], - RequestSense->Information[3], - RequestSense->CommandSpecificInformation[0], - RequestSense->CommandSpecificInformation[1], - RequestSense->CommandSpecificInformation[2], - RequestSense->CommandSpecificInformation[3]); - break; - case 'E': - if (Controller->SuppressEnclosureMessages) break; - sprintf(MessageBuffer, EventMessage, Event->LogicalUnit); - DAC960_Critical("Enclosure %d %s\n", Controller, - Event->TargetID, MessageBuffer); - break; - case 'C': - DAC960_Critical("Controller %s\n", Controller, EventMessage); - break; - default: - DAC960_Critical("Unknown Controller Event Code %04X\n", - Controller, Event->EventCode); - break; - } -} - - -/* - DAC960_V2_ReportProgress prints an appropriate progress message for - Logical Device Long Operations. -*/ - -static void DAC960_V2_ReportProgress(DAC960_Controller_T *Controller, - unsigned char *MessageString, - unsigned int LogicalDeviceNumber, - unsigned long BlocksCompleted, - unsigned long LogicalDeviceSize) -{ - Controller->EphemeralProgressMessage = true; - DAC960_Progress("%s in Progress: Logical Drive %d (/dev/rd/c%dd%d) " - "%d%% completed\n", Controller, - MessageString, - LogicalDeviceNumber, - Controller->ControllerNumber, - LogicalDeviceNumber, - (100 * (BlocksCompleted >> 7)) / (LogicalDeviceSize >> 7)); - Controller->EphemeralProgressMessage = false; -} - - -/* - DAC960_V2_ProcessCompletedCommand performs completion processing for Command - for DAC960 V2 Firmware Controllers. -*/ - -static void DAC960_V2_ProcessCompletedCommand(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - DAC960_CommandType_T CommandType = Command->CommandType; - DAC960_V2_CommandMailbox_T *CommandMailbox = &Command->V2.CommandMailbox; - DAC960_V2_IOCTL_Opcode_T IOCTLOpcode = CommandMailbox->Common.IOCTL_Opcode; - DAC960_V2_CommandOpcode_T CommandOpcode = CommandMailbox->SCSI_10.CommandOpcode; - DAC960_V2_CommandStatus_T CommandStatus = Command->V2.CommandStatus; - - if (CommandType == DAC960_ReadCommand || - CommandType == DAC960_WriteCommand) - { - -#ifdef FORCE_RETRY_DEBUG - CommandStatus = DAC960_V2_AbormalCompletion; -#endif - Command->V2.RequestSense->SenseKey = DAC960_SenseKey_MediumError; - - if (CommandStatus == DAC960_V2_NormalCompletion) { - - if (!DAC960_ProcessCompletedRequest(Command, true)) - BUG(); - - } else if (Command->V2.RequestSense->SenseKey == DAC960_SenseKey_MediumError) - { - /* - * break the command down into pieces and resubmit each - * piece, hoping that some of them will succeed. - */ - DAC960_queue_partial_rw(Command); - return; - } - else - { - if (Command->V2.RequestSense->SenseKey != DAC960_SenseKey_NotReady) - DAC960_V2_ReadWriteError(Command); - /* - Perform completion processing for all buffers in this I/O Request. - */ - (void)DAC960_ProcessCompletedRequest(Command, false); - } - } - else if (CommandType == DAC960_ReadRetryCommand || - CommandType == DAC960_WriteRetryCommand) - { - bool normal_completion; - -#ifdef FORCE_RETRY_FAILURE_DEBUG - static int retry_count = 1; -#endif - /* - Perform completion processing for the portion that was - retried, and submit the next portion, if any. - */ - normal_completion = true; - if (CommandStatus != DAC960_V2_NormalCompletion) { - normal_completion = false; - if (Command->V2.RequestSense->SenseKey != DAC960_SenseKey_NotReady) - DAC960_V2_ReadWriteError(Command); - } - -#ifdef FORCE_RETRY_FAILURE_DEBUG - if (!(++retry_count % 10000)) { - printk("V2 error retry failure test\n"); - normal_completion = false; - DAC960_V2_ReadWriteError(Command); - } -#endif - - if (!DAC960_ProcessCompletedRequest(Command, normal_completion)) { - DAC960_queue_partial_rw(Command); - return; - } - } - else if (CommandType == DAC960_MonitoringCommand) - { - if (Controller->ShutdownMonitoringTimer) - return; - if (IOCTLOpcode == DAC960_V2_GetControllerInfo) - { - DAC960_V2_ControllerInfo_T *NewControllerInfo = - Controller->V2.NewControllerInformation; - DAC960_V2_ControllerInfo_T *ControllerInfo = - &Controller->V2.ControllerInformation; - Controller->LogicalDriveCount = - NewControllerInfo->LogicalDevicesPresent; - Controller->V2.NeedLogicalDeviceInformation = true; - Controller->V2.NeedPhysicalDeviceInformation = true; - Controller->V2.StartLogicalDeviceInformationScan = true; - Controller->V2.StartPhysicalDeviceInformationScan = true; - Controller->MonitoringAlertMode = - (NewControllerInfo->LogicalDevicesCritical > 0 || - NewControllerInfo->LogicalDevicesOffline > 0 || - NewControllerInfo->PhysicalDisksCritical > 0 || - NewControllerInfo->PhysicalDisksOffline > 0); - memcpy(ControllerInfo, NewControllerInfo, - sizeof(DAC960_V2_ControllerInfo_T)); - } - else if (IOCTLOpcode == DAC960_V2_GetEvent) - { - if (CommandStatus == DAC960_V2_NormalCompletion) { - DAC960_V2_ReportEvent(Controller, Controller->V2.Event); - } - Controller->V2.NextEventSequenceNumber++; - } - else if (IOCTLOpcode == DAC960_V2_GetPhysicalDeviceInfoValid && - CommandStatus == DAC960_V2_NormalCompletion) - { - DAC960_V2_PhysicalDeviceInfo_T *NewPhysicalDeviceInfo = - Controller->V2.NewPhysicalDeviceInformation; - unsigned int PhysicalDeviceIndex = Controller->V2.PhysicalDeviceIndex; - DAC960_V2_PhysicalDeviceInfo_T *PhysicalDeviceInfo = - Controller->V2.PhysicalDeviceInformation[PhysicalDeviceIndex]; - DAC960_SCSI_Inquiry_UnitSerialNumber_T *InquiryUnitSerialNumber = - Controller->V2.InquiryUnitSerialNumber[PhysicalDeviceIndex]; - unsigned int DeviceIndex; - while (PhysicalDeviceInfo != NULL && - (NewPhysicalDeviceInfo->Channel > - PhysicalDeviceInfo->Channel || - (NewPhysicalDeviceInfo->Channel == - PhysicalDeviceInfo->Channel && - (NewPhysicalDeviceInfo->TargetID > - PhysicalDeviceInfo->TargetID || - (NewPhysicalDeviceInfo->TargetID == - PhysicalDeviceInfo->TargetID && - NewPhysicalDeviceInfo->LogicalUnit > - PhysicalDeviceInfo->LogicalUnit))))) - { - DAC960_Critical("Physical Device %d:%d No Longer Exists\n", - Controller, - PhysicalDeviceInfo->Channel, - PhysicalDeviceInfo->TargetID); - Controller->V2.PhysicalDeviceInformation - [PhysicalDeviceIndex] = NULL; - Controller->V2.InquiryUnitSerialNumber - [PhysicalDeviceIndex] = NULL; - kfree(PhysicalDeviceInfo); - kfree(InquiryUnitSerialNumber); - for (DeviceIndex = PhysicalDeviceIndex; - DeviceIndex < DAC960_V2_MaxPhysicalDevices - 1; - DeviceIndex++) - { - Controller->V2.PhysicalDeviceInformation[DeviceIndex] = - Controller->V2.PhysicalDeviceInformation[DeviceIndex+1]; - Controller->V2.InquiryUnitSerialNumber[DeviceIndex] = - Controller->V2.InquiryUnitSerialNumber[DeviceIndex+1]; - } - Controller->V2.PhysicalDeviceInformation - [DAC960_V2_MaxPhysicalDevices-1] = NULL; - Controller->V2.InquiryUnitSerialNumber - [DAC960_V2_MaxPhysicalDevices-1] = NULL; - PhysicalDeviceInfo = - Controller->V2.PhysicalDeviceInformation[PhysicalDeviceIndex]; - InquiryUnitSerialNumber = - Controller->V2.InquiryUnitSerialNumber[PhysicalDeviceIndex]; - } - if (PhysicalDeviceInfo == NULL || - (NewPhysicalDeviceInfo->Channel != - PhysicalDeviceInfo->Channel) || - (NewPhysicalDeviceInfo->TargetID != - PhysicalDeviceInfo->TargetID) || - (NewPhysicalDeviceInfo->LogicalUnit != - PhysicalDeviceInfo->LogicalUnit)) - { - PhysicalDeviceInfo = - kmalloc(sizeof(DAC960_V2_PhysicalDeviceInfo_T), GFP_ATOMIC); - InquiryUnitSerialNumber = - kmalloc(sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T), - GFP_ATOMIC); - if (InquiryUnitSerialNumber == NULL || - PhysicalDeviceInfo == NULL) - { - kfree(InquiryUnitSerialNumber); - InquiryUnitSerialNumber = NULL; - kfree(PhysicalDeviceInfo); - PhysicalDeviceInfo = NULL; - } - DAC960_Critical("Physical Device %d:%d Now Exists%s\n", - Controller, - NewPhysicalDeviceInfo->Channel, - NewPhysicalDeviceInfo->TargetID, - (PhysicalDeviceInfo != NULL - ? "" : " - Allocation Failed")); - if (PhysicalDeviceInfo != NULL) - { - memset(PhysicalDeviceInfo, 0, - sizeof(DAC960_V2_PhysicalDeviceInfo_T)); - PhysicalDeviceInfo->PhysicalDeviceState = - DAC960_V2_Device_InvalidState; - memset(InquiryUnitSerialNumber, 0, - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T)); - InquiryUnitSerialNumber->PeripheralDeviceType = 0x1F; - for (DeviceIndex = DAC960_V2_MaxPhysicalDevices - 1; - DeviceIndex > PhysicalDeviceIndex; - DeviceIndex--) - { - Controller->V2.PhysicalDeviceInformation[DeviceIndex] = - Controller->V2.PhysicalDeviceInformation[DeviceIndex-1]; - Controller->V2.InquiryUnitSerialNumber[DeviceIndex] = - Controller->V2.InquiryUnitSerialNumber[DeviceIndex-1]; - } - Controller->V2.PhysicalDeviceInformation - [PhysicalDeviceIndex] = - PhysicalDeviceInfo; - Controller->V2.InquiryUnitSerialNumber - [PhysicalDeviceIndex] = - InquiryUnitSerialNumber; - Controller->V2.NeedDeviceSerialNumberInformation = true; - } - } - if (PhysicalDeviceInfo != NULL) - { - if (NewPhysicalDeviceInfo->PhysicalDeviceState != - PhysicalDeviceInfo->PhysicalDeviceState) - DAC960_Critical( - "Physical Device %d:%d is now %s\n", Controller, - NewPhysicalDeviceInfo->Channel, - NewPhysicalDeviceInfo->TargetID, - (NewPhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_Online - ? "ONLINE" - : NewPhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_Rebuild - ? "REBUILD" - : NewPhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_Missing - ? "MISSING" - : NewPhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_Critical - ? "CRITICAL" - : NewPhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_Dead - ? "DEAD" - : NewPhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_SuspectedDead - ? "SUSPECTED-DEAD" - : NewPhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_CommandedOffline - ? "COMMANDED-OFFLINE" - : NewPhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_Standby - ? "STANDBY" : "UNKNOWN")); - if ((NewPhysicalDeviceInfo->ParityErrors != - PhysicalDeviceInfo->ParityErrors) || - (NewPhysicalDeviceInfo->SoftErrors != - PhysicalDeviceInfo->SoftErrors) || - (NewPhysicalDeviceInfo->HardErrors != - PhysicalDeviceInfo->HardErrors) || - (NewPhysicalDeviceInfo->MiscellaneousErrors != - PhysicalDeviceInfo->MiscellaneousErrors) || - (NewPhysicalDeviceInfo->CommandTimeouts != - PhysicalDeviceInfo->CommandTimeouts) || - (NewPhysicalDeviceInfo->Retries != - PhysicalDeviceInfo->Retries) || - (NewPhysicalDeviceInfo->Aborts != - PhysicalDeviceInfo->Aborts) || - (NewPhysicalDeviceInfo->PredictedFailuresDetected != - PhysicalDeviceInfo->PredictedFailuresDetected)) - { - DAC960_Critical("Physical Device %d:%d Errors: " - "Parity = %d, Soft = %d, " - "Hard = %d, Misc = %d\n", - Controller, - NewPhysicalDeviceInfo->Channel, - NewPhysicalDeviceInfo->TargetID, - NewPhysicalDeviceInfo->ParityErrors, - NewPhysicalDeviceInfo->SoftErrors, - NewPhysicalDeviceInfo->HardErrors, - NewPhysicalDeviceInfo->MiscellaneousErrors); - DAC960_Critical("Physical Device %d:%d Errors: " - "Timeouts = %d, Retries = %d, " - "Aborts = %d, Predicted = %d\n", - Controller, - NewPhysicalDeviceInfo->Channel, - NewPhysicalDeviceInfo->TargetID, - NewPhysicalDeviceInfo->CommandTimeouts, - NewPhysicalDeviceInfo->Retries, - NewPhysicalDeviceInfo->Aborts, - NewPhysicalDeviceInfo - ->PredictedFailuresDetected); - } - if ((PhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_Dead || - PhysicalDeviceInfo->PhysicalDeviceState - == DAC960_V2_Device_InvalidState) && - NewPhysicalDeviceInfo->PhysicalDeviceState - != DAC960_V2_Device_Dead) - Controller->V2.NeedDeviceSerialNumberInformation = true; - memcpy(PhysicalDeviceInfo, NewPhysicalDeviceInfo, - sizeof(DAC960_V2_PhysicalDeviceInfo_T)); - } - NewPhysicalDeviceInfo->LogicalUnit++; - Controller->V2.PhysicalDeviceIndex++; - } - else if (IOCTLOpcode == DAC960_V2_GetPhysicalDeviceInfoValid) - { - unsigned int DeviceIndex; - for (DeviceIndex = Controller->V2.PhysicalDeviceIndex; - DeviceIndex < DAC960_V2_MaxPhysicalDevices; - DeviceIndex++) - { - DAC960_V2_PhysicalDeviceInfo_T *PhysicalDeviceInfo = - Controller->V2.PhysicalDeviceInformation[DeviceIndex]; - DAC960_SCSI_Inquiry_UnitSerialNumber_T *InquiryUnitSerialNumber = - Controller->V2.InquiryUnitSerialNumber[DeviceIndex]; - if (PhysicalDeviceInfo == NULL) break; - DAC960_Critical("Physical Device %d:%d No Longer Exists\n", - Controller, - PhysicalDeviceInfo->Channel, - PhysicalDeviceInfo->TargetID); - Controller->V2.PhysicalDeviceInformation[DeviceIndex] = NULL; - Controller->V2.InquiryUnitSerialNumber[DeviceIndex] = NULL; - kfree(PhysicalDeviceInfo); - kfree(InquiryUnitSerialNumber); - } - Controller->V2.NeedPhysicalDeviceInformation = false; - } - else if (IOCTLOpcode == DAC960_V2_GetLogicalDeviceInfoValid && - CommandStatus == DAC960_V2_NormalCompletion) - { - DAC960_V2_LogicalDeviceInfo_T *NewLogicalDeviceInfo = - Controller->V2.NewLogicalDeviceInformation; - unsigned short LogicalDeviceNumber = - NewLogicalDeviceInfo->LogicalDeviceNumber; - DAC960_V2_LogicalDeviceInfo_T *LogicalDeviceInfo = - Controller->V2.LogicalDeviceInformation[LogicalDeviceNumber]; - if (LogicalDeviceInfo == NULL) - { - DAC960_V2_PhysicalDevice_T PhysicalDevice; - PhysicalDevice.Controller = 0; - PhysicalDevice.Channel = NewLogicalDeviceInfo->Channel; - PhysicalDevice.TargetID = NewLogicalDeviceInfo->TargetID; - PhysicalDevice.LogicalUnit = NewLogicalDeviceInfo->LogicalUnit; - Controller->V2.LogicalDriveToVirtualDevice[LogicalDeviceNumber] = - PhysicalDevice; - LogicalDeviceInfo = kmalloc(sizeof(DAC960_V2_LogicalDeviceInfo_T), - GFP_ATOMIC); - Controller->V2.LogicalDeviceInformation[LogicalDeviceNumber] = - LogicalDeviceInfo; - DAC960_Critical("Logical Drive %d (/dev/rd/c%dd%d) " - "Now Exists%s\n", Controller, - LogicalDeviceNumber, - Controller->ControllerNumber, - LogicalDeviceNumber, - (LogicalDeviceInfo != NULL - ? "" : " - Allocation Failed")); - if (LogicalDeviceInfo != NULL) - { - memset(LogicalDeviceInfo, 0, - sizeof(DAC960_V2_LogicalDeviceInfo_T)); - DAC960_ComputeGenericDiskInfo(Controller); - } - } - if (LogicalDeviceInfo != NULL) - { - unsigned long LogicalDeviceSize = - NewLogicalDeviceInfo->ConfigurableDeviceSize; - if (NewLogicalDeviceInfo->LogicalDeviceState != - LogicalDeviceInfo->LogicalDeviceState) - DAC960_Critical("Logical Drive %d (/dev/rd/c%dd%d) " - "is now %s\n", Controller, - LogicalDeviceNumber, - Controller->ControllerNumber, - LogicalDeviceNumber, - (NewLogicalDeviceInfo->LogicalDeviceState - == DAC960_V2_LogicalDevice_Online - ? "ONLINE" - : NewLogicalDeviceInfo->LogicalDeviceState - == DAC960_V2_LogicalDevice_Critical - ? "CRITICAL" : "OFFLINE")); - if ((NewLogicalDeviceInfo->SoftErrors != - LogicalDeviceInfo->SoftErrors) || - (NewLogicalDeviceInfo->CommandsFailed != - LogicalDeviceInfo->CommandsFailed) || - (NewLogicalDeviceInfo->DeferredWriteErrors != - LogicalDeviceInfo->DeferredWriteErrors)) - DAC960_Critical("Logical Drive %d (/dev/rd/c%dd%d) Errors: " - "Soft = %d, Failed = %d, Deferred Write = %d\n", - Controller, LogicalDeviceNumber, - Controller->ControllerNumber, - LogicalDeviceNumber, - NewLogicalDeviceInfo->SoftErrors, - NewLogicalDeviceInfo->CommandsFailed, - NewLogicalDeviceInfo->DeferredWriteErrors); - if (NewLogicalDeviceInfo->ConsistencyCheckInProgress) - DAC960_V2_ReportProgress(Controller, - "Consistency Check", - LogicalDeviceNumber, - NewLogicalDeviceInfo - ->ConsistencyCheckBlockNumber, - LogicalDeviceSize); - else if (NewLogicalDeviceInfo->RebuildInProgress) - DAC960_V2_ReportProgress(Controller, - "Rebuild", - LogicalDeviceNumber, - NewLogicalDeviceInfo - ->RebuildBlockNumber, - LogicalDeviceSize); - else if (NewLogicalDeviceInfo->BackgroundInitializationInProgress) - DAC960_V2_ReportProgress(Controller, - "Background Initialization", - LogicalDeviceNumber, - NewLogicalDeviceInfo - ->BackgroundInitializationBlockNumber, - LogicalDeviceSize); - else if (NewLogicalDeviceInfo->ForegroundInitializationInProgress) - DAC960_V2_ReportProgress(Controller, - "Foreground Initialization", - LogicalDeviceNumber, - NewLogicalDeviceInfo - ->ForegroundInitializationBlockNumber, - LogicalDeviceSize); - else if (NewLogicalDeviceInfo->DataMigrationInProgress) - DAC960_V2_ReportProgress(Controller, - "Data Migration", - LogicalDeviceNumber, - NewLogicalDeviceInfo - ->DataMigrationBlockNumber, - LogicalDeviceSize); - else if (NewLogicalDeviceInfo->PatrolOperationInProgress) - DAC960_V2_ReportProgress(Controller, - "Patrol Operation", - LogicalDeviceNumber, - NewLogicalDeviceInfo - ->PatrolOperationBlockNumber, - LogicalDeviceSize); - if (LogicalDeviceInfo->BackgroundInitializationInProgress && - !NewLogicalDeviceInfo->BackgroundInitializationInProgress) - DAC960_Progress("Logical Drive %d (/dev/rd/c%dd%d) " - "Background Initialization %s\n", - Controller, - LogicalDeviceNumber, - Controller->ControllerNumber, - LogicalDeviceNumber, - (NewLogicalDeviceInfo->LogicalDeviceControl - .LogicalDeviceInitialized - ? "Completed" : "Failed")); - memcpy(LogicalDeviceInfo, NewLogicalDeviceInfo, - sizeof(DAC960_V2_LogicalDeviceInfo_T)); - } - Controller->V2.LogicalDriveFoundDuringScan - [LogicalDeviceNumber] = true; - NewLogicalDeviceInfo->LogicalDeviceNumber++; - } - else if (IOCTLOpcode == DAC960_V2_GetLogicalDeviceInfoValid) - { - int LogicalDriveNumber; - for (LogicalDriveNumber = 0; - LogicalDriveNumber < DAC960_MaxLogicalDrives; - LogicalDriveNumber++) - { - DAC960_V2_LogicalDeviceInfo_T *LogicalDeviceInfo = - Controller->V2.LogicalDeviceInformation[LogicalDriveNumber]; - if (LogicalDeviceInfo == NULL || - Controller->V2.LogicalDriveFoundDuringScan - [LogicalDriveNumber]) - continue; - DAC960_Critical("Logical Drive %d (/dev/rd/c%dd%d) " - "No Longer Exists\n", Controller, - LogicalDriveNumber, - Controller->ControllerNumber, - LogicalDriveNumber); - Controller->V2.LogicalDeviceInformation - [LogicalDriveNumber] = NULL; - kfree(LogicalDeviceInfo); - Controller->LogicalDriveInitiallyAccessible - [LogicalDriveNumber] = false; - DAC960_ComputeGenericDiskInfo(Controller); - } - Controller->V2.NeedLogicalDeviceInformation = false; - } - else if (CommandOpcode == DAC960_V2_SCSI_10_Passthru) - { - DAC960_SCSI_Inquiry_UnitSerialNumber_T *InquiryUnitSerialNumber = - Controller->V2.InquiryUnitSerialNumber[Controller->V2.PhysicalDeviceIndex - 1]; - - if (CommandStatus != DAC960_V2_NormalCompletion) { - memset(InquiryUnitSerialNumber, - 0, sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T)); - InquiryUnitSerialNumber->PeripheralDeviceType = 0x1F; - } else - memcpy(InquiryUnitSerialNumber, - Controller->V2.NewInquiryUnitSerialNumber, - sizeof(DAC960_SCSI_Inquiry_UnitSerialNumber_T)); - - Controller->V2.NeedDeviceSerialNumberInformation = false; - } - - if (Controller->V2.HealthStatusBuffer->NextEventSequenceNumber - - Controller->V2.NextEventSequenceNumber > 0) - { - CommandMailbox->GetEvent.CommandOpcode = DAC960_V2_IOCTL; - CommandMailbox->GetEvent.DataTransferSize = sizeof(DAC960_V2_Event_T); - CommandMailbox->GetEvent.EventSequenceNumberHigh16 = - Controller->V2.NextEventSequenceNumber >> 16; - CommandMailbox->GetEvent.ControllerNumber = 0; - CommandMailbox->GetEvent.IOCTL_Opcode = - DAC960_V2_GetEvent; - CommandMailbox->GetEvent.EventSequenceNumberLow16 = - Controller->V2.NextEventSequenceNumber & 0xFFFF; - CommandMailbox->GetEvent.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentDataPointer = - Controller->V2.EventDMA; - CommandMailbox->GetEvent.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentByteCount = - CommandMailbox->GetEvent.DataTransferSize; - DAC960_QueueCommand(Command); - return; - } - if (Controller->V2.NeedPhysicalDeviceInformation) - { - if (Controller->V2.NeedDeviceSerialNumberInformation) - { - DAC960_SCSI_Inquiry_UnitSerialNumber_T *InquiryUnitSerialNumber = - Controller->V2.NewInquiryUnitSerialNumber; - InquiryUnitSerialNumber->PeripheralDeviceType = 0x1F; - - DAC960_V2_ConstructNewUnitSerialNumber(Controller, CommandMailbox, - Controller->V2.NewPhysicalDeviceInformation->Channel, - Controller->V2.NewPhysicalDeviceInformation->TargetID, - Controller->V2.NewPhysicalDeviceInformation->LogicalUnit - 1); - - - DAC960_QueueCommand(Command); - return; - } - if (Controller->V2.StartPhysicalDeviceInformationScan) - { - Controller->V2.PhysicalDeviceIndex = 0; - Controller->V2.NewPhysicalDeviceInformation->Channel = 0; - Controller->V2.NewPhysicalDeviceInformation->TargetID = 0; - Controller->V2.NewPhysicalDeviceInformation->LogicalUnit = 0; - Controller->V2.StartPhysicalDeviceInformationScan = false; - } - CommandMailbox->PhysicalDeviceInfo.CommandOpcode = DAC960_V2_IOCTL; - CommandMailbox->PhysicalDeviceInfo.DataTransferSize = - sizeof(DAC960_V2_PhysicalDeviceInfo_T); - CommandMailbox->PhysicalDeviceInfo.PhysicalDevice.LogicalUnit = - Controller->V2.NewPhysicalDeviceInformation->LogicalUnit; - CommandMailbox->PhysicalDeviceInfo.PhysicalDevice.TargetID = - Controller->V2.NewPhysicalDeviceInformation->TargetID; - CommandMailbox->PhysicalDeviceInfo.PhysicalDevice.Channel = - Controller->V2.NewPhysicalDeviceInformation->Channel; - CommandMailbox->PhysicalDeviceInfo.IOCTL_Opcode = - DAC960_V2_GetPhysicalDeviceInfoValid; - CommandMailbox->PhysicalDeviceInfo.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentDataPointer = - Controller->V2.NewPhysicalDeviceInformationDMA; - CommandMailbox->PhysicalDeviceInfo.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentByteCount = - CommandMailbox->PhysicalDeviceInfo.DataTransferSize; - DAC960_QueueCommand(Command); - return; - } - if (Controller->V2.NeedLogicalDeviceInformation) - { - if (Controller->V2.StartLogicalDeviceInformationScan) - { - int LogicalDriveNumber; - for (LogicalDriveNumber = 0; - LogicalDriveNumber < DAC960_MaxLogicalDrives; - LogicalDriveNumber++) - Controller->V2.LogicalDriveFoundDuringScan - [LogicalDriveNumber] = false; - Controller->V2.NewLogicalDeviceInformation->LogicalDeviceNumber = 0; - Controller->V2.StartLogicalDeviceInformationScan = false; - } - CommandMailbox->LogicalDeviceInfo.CommandOpcode = DAC960_V2_IOCTL; - CommandMailbox->LogicalDeviceInfo.DataTransferSize = - sizeof(DAC960_V2_LogicalDeviceInfo_T); - CommandMailbox->LogicalDeviceInfo.LogicalDevice.LogicalDeviceNumber = - Controller->V2.NewLogicalDeviceInformation->LogicalDeviceNumber; - CommandMailbox->LogicalDeviceInfo.IOCTL_Opcode = - DAC960_V2_GetLogicalDeviceInfoValid; - CommandMailbox->LogicalDeviceInfo.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentDataPointer = - Controller->V2.NewLogicalDeviceInformationDMA; - CommandMailbox->LogicalDeviceInfo.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentByteCount = - CommandMailbox->LogicalDeviceInfo.DataTransferSize; - DAC960_QueueCommand(Command); - return; - } - Controller->MonitoringTimerCount++; - Controller->MonitoringTimer.expires = - jiffies + DAC960_HealthStatusMonitoringInterval; - add_timer(&Controller->MonitoringTimer); - } - if (CommandType == DAC960_ImmediateCommand) - { - complete(Command->Completion); - Command->Completion = NULL; - return; - } - if (CommandType == DAC960_QueuedCommand) - { - DAC960_V2_KernelCommand_T *KernelCommand = Command->V2.KernelCommand; - KernelCommand->CommandStatus = CommandStatus; - KernelCommand->RequestSenseLength = Command->V2.RequestSenseLength; - KernelCommand->DataTransferLength = Command->V2.DataTransferResidue; - Command->V2.KernelCommand = NULL; - DAC960_DeallocateCommand(Command); - KernelCommand->CompletionFunction(KernelCommand); - return; - } - /* - Queue a Status Monitoring Command to the Controller using the just - completed Command if one was deferred previously due to lack of a - free Command when the Monitoring Timer Function was called. - */ - if (Controller->MonitoringCommandDeferred) - { - Controller->MonitoringCommandDeferred = false; - DAC960_V2_QueueMonitoringCommand(Command); - return; - } - /* - Deallocate the Command. - */ - DAC960_DeallocateCommand(Command); - /* - Wake up any processes waiting on a free Command. - */ - wake_up(&Controller->CommandWaitQueue); -} - -/* - DAC960_GEM_InterruptHandler handles hardware interrupts from DAC960 GEM Series - Controllers. -*/ - -static irqreturn_t DAC960_GEM_InterruptHandler(int IRQ_Channel, - void *DeviceIdentifier) -{ - DAC960_Controller_T *Controller = DeviceIdentifier; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - DAC960_V2_StatusMailbox_T *NextStatusMailbox; - unsigned long flags; - - spin_lock_irqsave(&Controller->queue_lock, flags); - DAC960_GEM_AcknowledgeInterrupt(ControllerBaseAddress); - NextStatusMailbox = Controller->V2.NextStatusMailbox; - while (NextStatusMailbox->Fields.CommandIdentifier > 0) - { - DAC960_V2_CommandIdentifier_T CommandIdentifier = - NextStatusMailbox->Fields.CommandIdentifier; - DAC960_Command_T *Command = Controller->Commands[CommandIdentifier-1]; - Command->V2.CommandStatus = NextStatusMailbox->Fields.CommandStatus; - Command->V2.RequestSenseLength = - NextStatusMailbox->Fields.RequestSenseLength; - Command->V2.DataTransferResidue = - NextStatusMailbox->Fields.DataTransferResidue; - NextStatusMailbox->Words[0] = 0; - if (++NextStatusMailbox > Controller->V2.LastStatusMailbox) - NextStatusMailbox = Controller->V2.FirstStatusMailbox; - DAC960_V2_ProcessCompletedCommand(Command); - } - Controller->V2.NextStatusMailbox = NextStatusMailbox; - /* - Attempt to remove additional I/O Requests from the Controller's - I/O Request Queue and queue them to the Controller. - */ - DAC960_ProcessRequest(Controller); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - return IRQ_HANDLED; -} - -/* - DAC960_BA_InterruptHandler handles hardware interrupts from DAC960 BA Series - Controllers. -*/ - -static irqreturn_t DAC960_BA_InterruptHandler(int IRQ_Channel, - void *DeviceIdentifier) -{ - DAC960_Controller_T *Controller = DeviceIdentifier; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - DAC960_V2_StatusMailbox_T *NextStatusMailbox; - unsigned long flags; - - spin_lock_irqsave(&Controller->queue_lock, flags); - DAC960_BA_AcknowledgeInterrupt(ControllerBaseAddress); - NextStatusMailbox = Controller->V2.NextStatusMailbox; - while (NextStatusMailbox->Fields.CommandIdentifier > 0) - { - DAC960_V2_CommandIdentifier_T CommandIdentifier = - NextStatusMailbox->Fields.CommandIdentifier; - DAC960_Command_T *Command = Controller->Commands[CommandIdentifier-1]; - Command->V2.CommandStatus = NextStatusMailbox->Fields.CommandStatus; - Command->V2.RequestSenseLength = - NextStatusMailbox->Fields.RequestSenseLength; - Command->V2.DataTransferResidue = - NextStatusMailbox->Fields.DataTransferResidue; - NextStatusMailbox->Words[0] = 0; - if (++NextStatusMailbox > Controller->V2.LastStatusMailbox) - NextStatusMailbox = Controller->V2.FirstStatusMailbox; - DAC960_V2_ProcessCompletedCommand(Command); - } - Controller->V2.NextStatusMailbox = NextStatusMailbox; - /* - Attempt to remove additional I/O Requests from the Controller's - I/O Request Queue and queue them to the Controller. - */ - DAC960_ProcessRequest(Controller); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - return IRQ_HANDLED; -} - - -/* - DAC960_LP_InterruptHandler handles hardware interrupts from DAC960 LP Series - Controllers. -*/ - -static irqreturn_t DAC960_LP_InterruptHandler(int IRQ_Channel, - void *DeviceIdentifier) -{ - DAC960_Controller_T *Controller = DeviceIdentifier; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - DAC960_V2_StatusMailbox_T *NextStatusMailbox; - unsigned long flags; - - spin_lock_irqsave(&Controller->queue_lock, flags); - DAC960_LP_AcknowledgeInterrupt(ControllerBaseAddress); - NextStatusMailbox = Controller->V2.NextStatusMailbox; - while (NextStatusMailbox->Fields.CommandIdentifier > 0) - { - DAC960_V2_CommandIdentifier_T CommandIdentifier = - NextStatusMailbox->Fields.CommandIdentifier; - DAC960_Command_T *Command = Controller->Commands[CommandIdentifier-1]; - Command->V2.CommandStatus = NextStatusMailbox->Fields.CommandStatus; - Command->V2.RequestSenseLength = - NextStatusMailbox->Fields.RequestSenseLength; - Command->V2.DataTransferResidue = - NextStatusMailbox->Fields.DataTransferResidue; - NextStatusMailbox->Words[0] = 0; - if (++NextStatusMailbox > Controller->V2.LastStatusMailbox) - NextStatusMailbox = Controller->V2.FirstStatusMailbox; - DAC960_V2_ProcessCompletedCommand(Command); - } - Controller->V2.NextStatusMailbox = NextStatusMailbox; - /* - Attempt to remove additional I/O Requests from the Controller's - I/O Request Queue and queue them to the Controller. - */ - DAC960_ProcessRequest(Controller); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - return IRQ_HANDLED; -} - - -/* - DAC960_LA_InterruptHandler handles hardware interrupts from DAC960 LA Series - Controllers. -*/ - -static irqreturn_t DAC960_LA_InterruptHandler(int IRQ_Channel, - void *DeviceIdentifier) -{ - DAC960_Controller_T *Controller = DeviceIdentifier; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - DAC960_V1_StatusMailbox_T *NextStatusMailbox; - unsigned long flags; - - spin_lock_irqsave(&Controller->queue_lock, flags); - DAC960_LA_AcknowledgeInterrupt(ControllerBaseAddress); - NextStatusMailbox = Controller->V1.NextStatusMailbox; - while (NextStatusMailbox->Fields.Valid) - { - DAC960_V1_CommandIdentifier_T CommandIdentifier = - NextStatusMailbox->Fields.CommandIdentifier; - DAC960_Command_T *Command = Controller->Commands[CommandIdentifier-1]; - Command->V1.CommandStatus = NextStatusMailbox->Fields.CommandStatus; - NextStatusMailbox->Word = 0; - if (++NextStatusMailbox > Controller->V1.LastStatusMailbox) - NextStatusMailbox = Controller->V1.FirstStatusMailbox; - DAC960_V1_ProcessCompletedCommand(Command); - } - Controller->V1.NextStatusMailbox = NextStatusMailbox; - /* - Attempt to remove additional I/O Requests from the Controller's - I/O Request Queue and queue them to the Controller. - */ - DAC960_ProcessRequest(Controller); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - return IRQ_HANDLED; -} - - -/* - DAC960_PG_InterruptHandler handles hardware interrupts from DAC960 PG Series - Controllers. -*/ - -static irqreturn_t DAC960_PG_InterruptHandler(int IRQ_Channel, - void *DeviceIdentifier) -{ - DAC960_Controller_T *Controller = DeviceIdentifier; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - DAC960_V1_StatusMailbox_T *NextStatusMailbox; - unsigned long flags; - - spin_lock_irqsave(&Controller->queue_lock, flags); - DAC960_PG_AcknowledgeInterrupt(ControllerBaseAddress); - NextStatusMailbox = Controller->V1.NextStatusMailbox; - while (NextStatusMailbox->Fields.Valid) - { - DAC960_V1_CommandIdentifier_T CommandIdentifier = - NextStatusMailbox->Fields.CommandIdentifier; - DAC960_Command_T *Command = Controller->Commands[CommandIdentifier-1]; - Command->V1.CommandStatus = NextStatusMailbox->Fields.CommandStatus; - NextStatusMailbox->Word = 0; - if (++NextStatusMailbox > Controller->V1.LastStatusMailbox) - NextStatusMailbox = Controller->V1.FirstStatusMailbox; - DAC960_V1_ProcessCompletedCommand(Command); - } - Controller->V1.NextStatusMailbox = NextStatusMailbox; - /* - Attempt to remove additional I/O Requests from the Controller's - I/O Request Queue and queue them to the Controller. - */ - DAC960_ProcessRequest(Controller); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - return IRQ_HANDLED; -} - - -/* - DAC960_PD_InterruptHandler handles hardware interrupts from DAC960 PD Series - Controllers. -*/ - -static irqreturn_t DAC960_PD_InterruptHandler(int IRQ_Channel, - void *DeviceIdentifier) -{ - DAC960_Controller_T *Controller = DeviceIdentifier; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - unsigned long flags; - - spin_lock_irqsave(&Controller->queue_lock, flags); - while (DAC960_PD_StatusAvailableP(ControllerBaseAddress)) - { - DAC960_V1_CommandIdentifier_T CommandIdentifier = - DAC960_PD_ReadStatusCommandIdentifier(ControllerBaseAddress); - DAC960_Command_T *Command = Controller->Commands[CommandIdentifier-1]; - Command->V1.CommandStatus = - DAC960_PD_ReadStatusRegister(ControllerBaseAddress); - DAC960_PD_AcknowledgeInterrupt(ControllerBaseAddress); - DAC960_PD_AcknowledgeStatus(ControllerBaseAddress); - DAC960_V1_ProcessCompletedCommand(Command); - } - /* - Attempt to remove additional I/O Requests from the Controller's - I/O Request Queue and queue them to the Controller. - */ - DAC960_ProcessRequest(Controller); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - return IRQ_HANDLED; -} - - -/* - DAC960_P_InterruptHandler handles hardware interrupts from DAC960 P Series - Controllers. - - Translations of DAC960_V1_Enquiry and DAC960_V1_GetDeviceState rely - on the data having been placed into DAC960_Controller_T, rather than - an arbitrary buffer. -*/ - -static irqreturn_t DAC960_P_InterruptHandler(int IRQ_Channel, - void *DeviceIdentifier) -{ - DAC960_Controller_T *Controller = DeviceIdentifier; - void __iomem *ControllerBaseAddress = Controller->BaseAddress; - unsigned long flags; - - spin_lock_irqsave(&Controller->queue_lock, flags); - while (DAC960_PD_StatusAvailableP(ControllerBaseAddress)) - { - DAC960_V1_CommandIdentifier_T CommandIdentifier = - DAC960_PD_ReadStatusCommandIdentifier(ControllerBaseAddress); - DAC960_Command_T *Command = Controller->Commands[CommandIdentifier-1]; - DAC960_V1_CommandMailbox_T *CommandMailbox = &Command->V1.CommandMailbox; - DAC960_V1_CommandOpcode_T CommandOpcode = - CommandMailbox->Common.CommandOpcode; - Command->V1.CommandStatus = - DAC960_PD_ReadStatusRegister(ControllerBaseAddress); - DAC960_PD_AcknowledgeInterrupt(ControllerBaseAddress); - DAC960_PD_AcknowledgeStatus(ControllerBaseAddress); - switch (CommandOpcode) - { - case DAC960_V1_Enquiry_Old: - Command->V1.CommandMailbox.Common.CommandOpcode = DAC960_V1_Enquiry; - DAC960_P_To_PD_TranslateEnquiry(Controller->V1.NewEnquiry); - break; - case DAC960_V1_GetDeviceState_Old: - Command->V1.CommandMailbox.Common.CommandOpcode = - DAC960_V1_GetDeviceState; - DAC960_P_To_PD_TranslateDeviceState(Controller->V1.NewDeviceState); - break; - case DAC960_V1_Read_Old: - Command->V1.CommandMailbox.Common.CommandOpcode = DAC960_V1_Read; - DAC960_P_To_PD_TranslateReadWriteCommand(CommandMailbox); - break; - case DAC960_V1_Write_Old: - Command->V1.CommandMailbox.Common.CommandOpcode = DAC960_V1_Write; - DAC960_P_To_PD_TranslateReadWriteCommand(CommandMailbox); - break; - case DAC960_V1_ReadWithScatterGather_Old: - Command->V1.CommandMailbox.Common.CommandOpcode = - DAC960_V1_ReadWithScatterGather; - DAC960_P_To_PD_TranslateReadWriteCommand(CommandMailbox); - break; - case DAC960_V1_WriteWithScatterGather_Old: - Command->V1.CommandMailbox.Common.CommandOpcode = - DAC960_V1_WriteWithScatterGather; - DAC960_P_To_PD_TranslateReadWriteCommand(CommandMailbox); - break; - default: - break; - } - DAC960_V1_ProcessCompletedCommand(Command); - } - /* - Attempt to remove additional I/O Requests from the Controller's - I/O Request Queue and queue them to the Controller. - */ - DAC960_ProcessRequest(Controller); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - return IRQ_HANDLED; -} - - -/* - DAC960_V1_QueueMonitoringCommand queues a Monitoring Command to DAC960 V1 - Firmware Controllers. -*/ - -static void DAC960_V1_QueueMonitoringCommand(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - DAC960_V1_CommandMailbox_T *CommandMailbox = &Command->V1.CommandMailbox; - DAC960_V1_ClearCommand(Command); - Command->CommandType = DAC960_MonitoringCommand; - CommandMailbox->Type3.CommandOpcode = DAC960_V1_Enquiry; - CommandMailbox->Type3.BusAddress = Controller->V1.NewEnquiryDMA; - DAC960_QueueCommand(Command); -} - - -/* - DAC960_V2_QueueMonitoringCommand queues a Monitoring Command to DAC960 V2 - Firmware Controllers. -*/ - -static void DAC960_V2_QueueMonitoringCommand(DAC960_Command_T *Command) -{ - DAC960_Controller_T *Controller = Command->Controller; - DAC960_V2_CommandMailbox_T *CommandMailbox = &Command->V2.CommandMailbox; - DAC960_V2_ClearCommand(Command); - Command->CommandType = DAC960_MonitoringCommand; - CommandMailbox->ControllerInfo.CommandOpcode = DAC960_V2_IOCTL; - CommandMailbox->ControllerInfo.CommandControlBits - .DataTransferControllerToHost = true; - CommandMailbox->ControllerInfo.CommandControlBits - .NoAutoRequestSense = true; - CommandMailbox->ControllerInfo.DataTransferSize = - sizeof(DAC960_V2_ControllerInfo_T); - CommandMailbox->ControllerInfo.ControllerNumber = 0; - CommandMailbox->ControllerInfo.IOCTL_Opcode = DAC960_V2_GetControllerInfo; - CommandMailbox->ControllerInfo.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentDataPointer = - Controller->V2.NewControllerInformationDMA; - CommandMailbox->ControllerInfo.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentByteCount = - CommandMailbox->ControllerInfo.DataTransferSize; - DAC960_QueueCommand(Command); -} - - -/* - DAC960_MonitoringTimerFunction is the timer function for monitoring - the status of DAC960 Controllers. -*/ - -static void DAC960_MonitoringTimerFunction(struct timer_list *t) -{ - DAC960_Controller_T *Controller = from_timer(Controller, t, MonitoringTimer); - DAC960_Command_T *Command; - unsigned long flags; - - if (Controller->FirmwareType == DAC960_V1_Controller) - { - spin_lock_irqsave(&Controller->queue_lock, flags); - /* - Queue a Status Monitoring Command to Controller. - */ - Command = DAC960_AllocateCommand(Controller); - if (Command != NULL) - DAC960_V1_QueueMonitoringCommand(Command); - else Controller->MonitoringCommandDeferred = true; - spin_unlock_irqrestore(&Controller->queue_lock, flags); - } - else - { - DAC960_V2_ControllerInfo_T *ControllerInfo = - &Controller->V2.ControllerInformation; - unsigned int StatusChangeCounter = - Controller->V2.HealthStatusBuffer->StatusChangeCounter; - bool ForceMonitoringCommand = false; - if (time_after(jiffies, Controller->SecondaryMonitoringTime - + DAC960_SecondaryMonitoringInterval)) - { - int LogicalDriveNumber; - for (LogicalDriveNumber = 0; - LogicalDriveNumber < DAC960_MaxLogicalDrives; - LogicalDriveNumber++) - { - DAC960_V2_LogicalDeviceInfo_T *LogicalDeviceInfo = - Controller->V2.LogicalDeviceInformation[LogicalDriveNumber]; - if (LogicalDeviceInfo == NULL) continue; - if (!LogicalDeviceInfo->LogicalDeviceControl - .LogicalDeviceInitialized) - { - ForceMonitoringCommand = true; - break; - } - } - Controller->SecondaryMonitoringTime = jiffies; - } - if (StatusChangeCounter == Controller->V2.StatusChangeCounter && - Controller->V2.HealthStatusBuffer->NextEventSequenceNumber - == Controller->V2.NextEventSequenceNumber && - (ControllerInfo->BackgroundInitializationsActive + - ControllerInfo->LogicalDeviceInitializationsActive + - ControllerInfo->PhysicalDeviceInitializationsActive + - ControllerInfo->ConsistencyChecksActive + - ControllerInfo->RebuildsActive + - ControllerInfo->OnlineExpansionsActive == 0 || - time_before(jiffies, Controller->PrimaryMonitoringTime - + DAC960_MonitoringTimerInterval)) && - !ForceMonitoringCommand) - { - Controller->MonitoringTimer.expires = - jiffies + DAC960_HealthStatusMonitoringInterval; - add_timer(&Controller->MonitoringTimer); - return; - } - Controller->V2.StatusChangeCounter = StatusChangeCounter; - Controller->PrimaryMonitoringTime = jiffies; - - spin_lock_irqsave(&Controller->queue_lock, flags); - /* - Queue a Status Monitoring Command to Controller. - */ - Command = DAC960_AllocateCommand(Controller); - if (Command != NULL) - DAC960_V2_QueueMonitoringCommand(Command); - else Controller->MonitoringCommandDeferred = true; - spin_unlock_irqrestore(&Controller->queue_lock, flags); - /* - Wake up any processes waiting on a Health Status Buffer change. - */ - wake_up(&Controller->HealthStatusWaitQueue); - } -} - -/* - DAC960_CheckStatusBuffer verifies that there is room to hold ByteCount - additional bytes in the Combined Status Buffer and grows the buffer if - necessary. It returns true if there is enough room and false otherwise. -*/ - -static bool DAC960_CheckStatusBuffer(DAC960_Controller_T *Controller, - unsigned int ByteCount) -{ - unsigned char *NewStatusBuffer; - if (Controller->InitialStatusLength + 1 + - Controller->CurrentStatusLength + ByteCount + 1 <= - Controller->CombinedStatusBufferLength) - return true; - if (Controller->CombinedStatusBufferLength == 0) - { - unsigned int NewStatusBufferLength = DAC960_InitialStatusBufferSize; - while (NewStatusBufferLength < ByteCount) - NewStatusBufferLength *= 2; - Controller->CombinedStatusBuffer = kmalloc(NewStatusBufferLength, - GFP_ATOMIC); - if (Controller->CombinedStatusBuffer == NULL) return false; - Controller->CombinedStatusBufferLength = NewStatusBufferLength; - return true; - } - NewStatusBuffer = kmalloc_array(2, Controller->CombinedStatusBufferLength, - GFP_ATOMIC); - if (NewStatusBuffer == NULL) - { - DAC960_Warning("Unable to expand Combined Status Buffer - Truncating\n", - Controller); - return false; - } - memcpy(NewStatusBuffer, Controller->CombinedStatusBuffer, - Controller->CombinedStatusBufferLength); - kfree(Controller->CombinedStatusBuffer); - Controller->CombinedStatusBuffer = NewStatusBuffer; - Controller->CombinedStatusBufferLength *= 2; - Controller->CurrentStatusBuffer = - &NewStatusBuffer[Controller->InitialStatusLength + 1]; - return true; -} - - -/* - DAC960_Message prints Driver Messages. -*/ - -static void DAC960_Message(DAC960_MessageLevel_T MessageLevel, - unsigned char *Format, - DAC960_Controller_T *Controller, - ...) -{ - static unsigned char Buffer[DAC960_LineBufferSize]; - static bool BeginningOfLine = true; - va_list Arguments; - int Length = 0; - va_start(Arguments, Controller); - Length = vsprintf(Buffer, Format, Arguments); - va_end(Arguments); - if (Controller == NULL) - printk("%sDAC960#%d: %s", DAC960_MessageLevelMap[MessageLevel], - DAC960_ControllerCount, Buffer); - else if (MessageLevel == DAC960_AnnounceLevel || - MessageLevel == DAC960_InfoLevel) - { - if (!Controller->ControllerInitialized) - { - if (DAC960_CheckStatusBuffer(Controller, Length)) - { - strcpy(&Controller->CombinedStatusBuffer - [Controller->InitialStatusLength], - Buffer); - Controller->InitialStatusLength += Length; - Controller->CurrentStatusBuffer = - &Controller->CombinedStatusBuffer - [Controller->InitialStatusLength + 1]; - } - if (MessageLevel == DAC960_AnnounceLevel) - { - static int AnnouncementLines = 0; - if (++AnnouncementLines <= 2) - printk("%sDAC960: %s", DAC960_MessageLevelMap[MessageLevel], - Buffer); - } - else - { - if (BeginningOfLine) - { - if (Buffer[0] != '\n' || Length > 1) - printk("%sDAC960#%d: %s", - DAC960_MessageLevelMap[MessageLevel], - Controller->ControllerNumber, Buffer); - } - else printk("%s", Buffer); - } - } - else if (DAC960_CheckStatusBuffer(Controller, Length)) - { - strcpy(&Controller->CurrentStatusBuffer[ - Controller->CurrentStatusLength], Buffer); - Controller->CurrentStatusLength += Length; - } - } - else if (MessageLevel == DAC960_ProgressLevel) - { - strcpy(Controller->ProgressBuffer, Buffer); - Controller->ProgressBufferLength = Length; - if (Controller->EphemeralProgressMessage) - { - if (time_after_eq(jiffies, Controller->LastProgressReportTime - + DAC960_ProgressReportingInterval)) - { - printk("%sDAC960#%d: %s", DAC960_MessageLevelMap[MessageLevel], - Controller->ControllerNumber, Buffer); - Controller->LastProgressReportTime = jiffies; - } - } - else printk("%sDAC960#%d: %s", DAC960_MessageLevelMap[MessageLevel], - Controller->ControllerNumber, Buffer); - } - else if (MessageLevel == DAC960_UserCriticalLevel) - { - strcpy(&Controller->UserStatusBuffer[Controller->UserStatusLength], - Buffer); - Controller->UserStatusLength += Length; - if (Buffer[0] != '\n' || Length > 1) - printk("%sDAC960#%d: %s", DAC960_MessageLevelMap[MessageLevel], - Controller->ControllerNumber, Buffer); - } - else - { - if (BeginningOfLine) - printk("%sDAC960#%d: %s", DAC960_MessageLevelMap[MessageLevel], - Controller->ControllerNumber, Buffer); - else printk("%s", Buffer); - } - BeginningOfLine = (Buffer[Length-1] == '\n'); -} - - -/* - DAC960_ParsePhysicalDevice parses spaces followed by a Physical Device - Channel:TargetID specification from a User Command string. It updates - Channel and TargetID and returns true on success and false on failure. -*/ - -static bool DAC960_ParsePhysicalDevice(DAC960_Controller_T *Controller, - char *UserCommandString, - unsigned char *Channel, - unsigned char *TargetID) -{ - char *NewUserCommandString = UserCommandString; - unsigned long XChannel, XTargetID; - while (*UserCommandString == ' ') UserCommandString++; - if (UserCommandString == NewUserCommandString) - return false; - XChannel = simple_strtoul(UserCommandString, &NewUserCommandString, 10); - if (NewUserCommandString == UserCommandString || - *NewUserCommandString != ':' || - XChannel >= Controller->Channels) - return false; - UserCommandString = ++NewUserCommandString; - XTargetID = simple_strtoul(UserCommandString, &NewUserCommandString, 10); - if (NewUserCommandString == UserCommandString || - *NewUserCommandString != '\0' || - XTargetID >= Controller->Targets) - return false; - *Channel = XChannel; - *TargetID = XTargetID; - return true; -} - - -/* - DAC960_ParseLogicalDrive parses spaces followed by a Logical Drive Number - specification from a User Command string. It updates LogicalDriveNumber and - returns true on success and false on failure. -*/ - -static bool DAC960_ParseLogicalDrive(DAC960_Controller_T *Controller, - char *UserCommandString, - unsigned char *LogicalDriveNumber) -{ - char *NewUserCommandString = UserCommandString; - unsigned long XLogicalDriveNumber; - while (*UserCommandString == ' ') UserCommandString++; - if (UserCommandString == NewUserCommandString) - return false; - XLogicalDriveNumber = - simple_strtoul(UserCommandString, &NewUserCommandString, 10); - if (NewUserCommandString == UserCommandString || - *NewUserCommandString != '\0' || - XLogicalDriveNumber > DAC960_MaxLogicalDrives - 1) - return false; - *LogicalDriveNumber = XLogicalDriveNumber; - return true; -} - - -/* - DAC960_V1_SetDeviceState sets the Device State for a Physical Device for - DAC960 V1 Firmware Controllers. -*/ - -static void DAC960_V1_SetDeviceState(DAC960_Controller_T *Controller, - DAC960_Command_T *Command, - unsigned char Channel, - unsigned char TargetID, - DAC960_V1_PhysicalDeviceState_T - DeviceState, - const unsigned char *DeviceStateString) -{ - DAC960_V1_CommandMailbox_T *CommandMailbox = &Command->V1.CommandMailbox; - CommandMailbox->Type3D.CommandOpcode = DAC960_V1_StartDevice; - CommandMailbox->Type3D.Channel = Channel; - CommandMailbox->Type3D.TargetID = TargetID; - CommandMailbox->Type3D.DeviceState = DeviceState; - CommandMailbox->Type3D.Modifier = 0; - DAC960_ExecuteCommand(Command); - switch (Command->V1.CommandStatus) - { - case DAC960_V1_NormalCompletion: - DAC960_UserCritical("%s of Physical Device %d:%d Succeeded\n", Controller, - DeviceStateString, Channel, TargetID); - break; - case DAC960_V1_UnableToStartDevice: - DAC960_UserCritical("%s of Physical Device %d:%d Failed - " - "Unable to Start Device\n", Controller, - DeviceStateString, Channel, TargetID); - break; - case DAC960_V1_NoDeviceAtAddress: - DAC960_UserCritical("%s of Physical Device %d:%d Failed - " - "No Device at Address\n", Controller, - DeviceStateString, Channel, TargetID); - break; - case DAC960_V1_InvalidChannelOrTargetOrModifier: - DAC960_UserCritical("%s of Physical Device %d:%d Failed - " - "Invalid Channel or Target or Modifier\n", - Controller, DeviceStateString, Channel, TargetID); - break; - case DAC960_V1_ChannelBusy: - DAC960_UserCritical("%s of Physical Device %d:%d Failed - " - "Channel Busy\n", Controller, - DeviceStateString, Channel, TargetID); - break; - default: - DAC960_UserCritical("%s of Physical Device %d:%d Failed - " - "Unexpected Status %04X\n", Controller, - DeviceStateString, Channel, TargetID, - Command->V1.CommandStatus); - break; - } -} - - -/* - DAC960_V1_ExecuteUserCommand executes a User Command for DAC960 V1 Firmware - Controllers. -*/ - -static bool DAC960_V1_ExecuteUserCommand(DAC960_Controller_T *Controller, - unsigned char *UserCommand) -{ - DAC960_Command_T *Command; - DAC960_V1_CommandMailbox_T *CommandMailbox; - unsigned long flags; - unsigned char Channel, TargetID, LogicalDriveNumber; - - spin_lock_irqsave(&Controller->queue_lock, flags); - while ((Command = DAC960_AllocateCommand(Controller)) == NULL) - DAC960_WaitForCommand(Controller); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - Controller->UserStatusLength = 0; - DAC960_V1_ClearCommand(Command); - Command->CommandType = DAC960_ImmediateCommand; - CommandMailbox = &Command->V1.CommandMailbox; - if (strcmp(UserCommand, "flush-cache") == 0) - { - CommandMailbox->Type3.CommandOpcode = DAC960_V1_Flush; - DAC960_ExecuteCommand(Command); - DAC960_UserCritical("Cache Flush Completed\n", Controller); - } - else if (strncmp(UserCommand, "kill", 4) == 0 && - DAC960_ParsePhysicalDevice(Controller, &UserCommand[4], - &Channel, &TargetID)) - { - DAC960_V1_DeviceState_T *DeviceState = - &Controller->V1.DeviceState[Channel][TargetID]; - if (DeviceState->Present && - DeviceState->DeviceType == DAC960_V1_DiskType && - DeviceState->DeviceState != DAC960_V1_Device_Dead) - DAC960_V1_SetDeviceState(Controller, Command, Channel, TargetID, - DAC960_V1_Device_Dead, "Kill"); - else DAC960_UserCritical("Kill of Physical Device %d:%d Illegal\n", - Controller, Channel, TargetID); - } - else if (strncmp(UserCommand, "make-online", 11) == 0 && - DAC960_ParsePhysicalDevice(Controller, &UserCommand[11], - &Channel, &TargetID)) - { - DAC960_V1_DeviceState_T *DeviceState = - &Controller->V1.DeviceState[Channel][TargetID]; - if (DeviceState->Present && - DeviceState->DeviceType == DAC960_V1_DiskType && - DeviceState->DeviceState == DAC960_V1_Device_Dead) - DAC960_V1_SetDeviceState(Controller, Command, Channel, TargetID, - DAC960_V1_Device_Online, "Make Online"); - else DAC960_UserCritical("Make Online of Physical Device %d:%d Illegal\n", - Controller, Channel, TargetID); - - } - else if (strncmp(UserCommand, "make-standby", 12) == 0 && - DAC960_ParsePhysicalDevice(Controller, &UserCommand[12], - &Channel, &TargetID)) - { - DAC960_V1_DeviceState_T *DeviceState = - &Controller->V1.DeviceState[Channel][TargetID]; - if (DeviceState->Present && - DeviceState->DeviceType == DAC960_V1_DiskType && - DeviceState->DeviceState == DAC960_V1_Device_Dead) - DAC960_V1_SetDeviceState(Controller, Command, Channel, TargetID, - DAC960_V1_Device_Standby, "Make Standby"); - else DAC960_UserCritical("Make Standby of Physical " - "Device %d:%d Illegal\n", - Controller, Channel, TargetID); - } - else if (strncmp(UserCommand, "rebuild", 7) == 0 && - DAC960_ParsePhysicalDevice(Controller, &UserCommand[7], - &Channel, &TargetID)) - { - CommandMailbox->Type3D.CommandOpcode = DAC960_V1_RebuildAsync; - CommandMailbox->Type3D.Channel = Channel; - CommandMailbox->Type3D.TargetID = TargetID; - DAC960_ExecuteCommand(Command); - switch (Command->V1.CommandStatus) - { - case DAC960_V1_NormalCompletion: - DAC960_UserCritical("Rebuild of Physical Device %d:%d Initiated\n", - Controller, Channel, TargetID); - break; - case DAC960_V1_AttemptToRebuildOnlineDrive: - DAC960_UserCritical("Rebuild of Physical Device %d:%d Failed - " - "Attempt to Rebuild Online or " - "Unresponsive Drive\n", - Controller, Channel, TargetID); - break; - case DAC960_V1_NewDiskFailedDuringRebuild: - DAC960_UserCritical("Rebuild of Physical Device %d:%d Failed - " - "New Disk Failed During Rebuild\n", - Controller, Channel, TargetID); - break; - case DAC960_V1_InvalidDeviceAddress: - DAC960_UserCritical("Rebuild of Physical Device %d:%d Failed - " - "Invalid Device Address\n", - Controller, Channel, TargetID); - break; - case DAC960_V1_RebuildOrCheckAlreadyInProgress: - DAC960_UserCritical("Rebuild of Physical Device %d:%d Failed - " - "Rebuild or Consistency Check Already " - "in Progress\n", Controller, Channel, TargetID); - break; - default: - DAC960_UserCritical("Rebuild of Physical Device %d:%d Failed - " - "Unexpected Status %04X\n", Controller, - Channel, TargetID, Command->V1.CommandStatus); - break; - } - } - else if (strncmp(UserCommand, "check-consistency", 17) == 0 && - DAC960_ParseLogicalDrive(Controller, &UserCommand[17], - &LogicalDriveNumber)) - { - CommandMailbox->Type3C.CommandOpcode = DAC960_V1_CheckConsistencyAsync; - CommandMailbox->Type3C.LogicalDriveNumber = LogicalDriveNumber; - CommandMailbox->Type3C.AutoRestore = true; - DAC960_ExecuteCommand(Command); - switch (Command->V1.CommandStatus) - { - case DAC960_V1_NormalCompletion: - DAC960_UserCritical("Consistency Check of Logical Drive %d " - "(/dev/rd/c%dd%d) Initiated\n", - Controller, LogicalDriveNumber, - Controller->ControllerNumber, - LogicalDriveNumber); - break; - case DAC960_V1_DependentDiskIsDead: - DAC960_UserCritical("Consistency Check of Logical Drive %d " - "(/dev/rd/c%dd%d) Failed - " - "Dependent Physical Device is DEAD\n", - Controller, LogicalDriveNumber, - Controller->ControllerNumber, - LogicalDriveNumber); - break; - case DAC960_V1_InvalidOrNonredundantLogicalDrive: - DAC960_UserCritical("Consistency Check of Logical Drive %d " - "(/dev/rd/c%dd%d) Failed - " - "Invalid or Nonredundant Logical Drive\n", - Controller, LogicalDriveNumber, - Controller->ControllerNumber, - LogicalDriveNumber); - break; - case DAC960_V1_RebuildOrCheckAlreadyInProgress: - DAC960_UserCritical("Consistency Check of Logical Drive %d " - "(/dev/rd/c%dd%d) Failed - Rebuild or " - "Consistency Check Already in Progress\n", - Controller, LogicalDriveNumber, - Controller->ControllerNumber, - LogicalDriveNumber); - break; - default: - DAC960_UserCritical("Consistency Check of Logical Drive %d " - "(/dev/rd/c%dd%d) Failed - " - "Unexpected Status %04X\n", - Controller, LogicalDriveNumber, - Controller->ControllerNumber, - LogicalDriveNumber, Command->V1.CommandStatus); - break; - } - } - else if (strcmp(UserCommand, "cancel-rebuild") == 0 || - strcmp(UserCommand, "cancel-consistency-check") == 0) - { - /* - the OldRebuildRateConstant is never actually used - once its value is retrieved from the controller. - */ - unsigned char *OldRebuildRateConstant; - dma_addr_t OldRebuildRateConstantDMA; - - OldRebuildRateConstant = pci_alloc_consistent( Controller->PCIDevice, - sizeof(char), &OldRebuildRateConstantDMA); - if (OldRebuildRateConstant == NULL) { - DAC960_UserCritical("Cancellation of Rebuild or " - "Consistency Check Failed - " - "Out of Memory", - Controller); - goto failure; - } - CommandMailbox->Type3R.CommandOpcode = DAC960_V1_RebuildControl; - CommandMailbox->Type3R.RebuildRateConstant = 0xFF; - CommandMailbox->Type3R.BusAddress = OldRebuildRateConstantDMA; - DAC960_ExecuteCommand(Command); - switch (Command->V1.CommandStatus) - { - case DAC960_V1_NormalCompletion: - DAC960_UserCritical("Rebuild or Consistency Check Cancelled\n", - Controller); - break; - default: - DAC960_UserCritical("Cancellation of Rebuild or " - "Consistency Check Failed - " - "Unexpected Status %04X\n", - Controller, Command->V1.CommandStatus); - break; - } -failure: - pci_free_consistent(Controller->PCIDevice, sizeof(char), - OldRebuildRateConstant, OldRebuildRateConstantDMA); - } - else DAC960_UserCritical("Illegal User Command: '%s'\n", - Controller, UserCommand); - - spin_lock_irqsave(&Controller->queue_lock, flags); - DAC960_DeallocateCommand(Command); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - return true; -} - - -/* - DAC960_V2_TranslatePhysicalDevice translates a Physical Device Channel and - TargetID into a Logical Device. It returns true on success and false - on failure. -*/ - -static bool DAC960_V2_TranslatePhysicalDevice(DAC960_Command_T *Command, - unsigned char Channel, - unsigned char TargetID, - unsigned short - *LogicalDeviceNumber) -{ - DAC960_V2_CommandMailbox_T SavedCommandMailbox, *CommandMailbox; - DAC960_Controller_T *Controller = Command->Controller; - - CommandMailbox = &Command->V2.CommandMailbox; - memcpy(&SavedCommandMailbox, CommandMailbox, - sizeof(DAC960_V2_CommandMailbox_T)); - - CommandMailbox->PhysicalDeviceInfo.CommandOpcode = DAC960_V2_IOCTL; - CommandMailbox->PhysicalDeviceInfo.CommandControlBits - .DataTransferControllerToHost = true; - CommandMailbox->PhysicalDeviceInfo.CommandControlBits - .NoAutoRequestSense = true; - CommandMailbox->PhysicalDeviceInfo.DataTransferSize = - sizeof(DAC960_V2_PhysicalToLogicalDevice_T); - CommandMailbox->PhysicalDeviceInfo.PhysicalDevice.TargetID = TargetID; - CommandMailbox->PhysicalDeviceInfo.PhysicalDevice.Channel = Channel; - CommandMailbox->PhysicalDeviceInfo.IOCTL_Opcode = - DAC960_V2_TranslatePhysicalToLogicalDevice; - CommandMailbox->Common.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentDataPointer = - Controller->V2.PhysicalToLogicalDeviceDMA; - CommandMailbox->Common.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentByteCount = - CommandMailbox->Common.DataTransferSize; - - DAC960_ExecuteCommand(Command); - *LogicalDeviceNumber = Controller->V2.PhysicalToLogicalDevice->LogicalDeviceNumber; - - memcpy(CommandMailbox, &SavedCommandMailbox, - sizeof(DAC960_V2_CommandMailbox_T)); - return (Command->V2.CommandStatus == DAC960_V2_NormalCompletion); -} - - -/* - DAC960_V2_ExecuteUserCommand executes a User Command for DAC960 V2 Firmware - Controllers. -*/ - -static bool DAC960_V2_ExecuteUserCommand(DAC960_Controller_T *Controller, - unsigned char *UserCommand) -{ - DAC960_Command_T *Command; - DAC960_V2_CommandMailbox_T *CommandMailbox; - unsigned long flags; - unsigned char Channel, TargetID, LogicalDriveNumber; - unsigned short LogicalDeviceNumber; - - spin_lock_irqsave(&Controller->queue_lock, flags); - while ((Command = DAC960_AllocateCommand(Controller)) == NULL) - DAC960_WaitForCommand(Controller); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - Controller->UserStatusLength = 0; - DAC960_V2_ClearCommand(Command); - Command->CommandType = DAC960_ImmediateCommand; - CommandMailbox = &Command->V2.CommandMailbox; - CommandMailbox->Common.CommandOpcode = DAC960_V2_IOCTL; - CommandMailbox->Common.CommandControlBits.DataTransferControllerToHost = true; - CommandMailbox->Common.CommandControlBits.NoAutoRequestSense = true; - if (strcmp(UserCommand, "flush-cache") == 0) - { - CommandMailbox->DeviceOperation.IOCTL_Opcode = DAC960_V2_PauseDevice; - CommandMailbox->DeviceOperation.OperationDevice = - DAC960_V2_RAID_Controller; - DAC960_ExecuteCommand(Command); - DAC960_UserCritical("Cache Flush Completed\n", Controller); - } - else if (strncmp(UserCommand, "kill", 4) == 0 && - DAC960_ParsePhysicalDevice(Controller, &UserCommand[4], - &Channel, &TargetID) && - DAC960_V2_TranslatePhysicalDevice(Command, Channel, TargetID, - &LogicalDeviceNumber)) - { - CommandMailbox->SetDeviceState.LogicalDevice.LogicalDeviceNumber = - LogicalDeviceNumber; - CommandMailbox->SetDeviceState.IOCTL_Opcode = - DAC960_V2_SetDeviceState; - CommandMailbox->SetDeviceState.DeviceState.PhysicalDeviceState = - DAC960_V2_Device_Dead; - DAC960_ExecuteCommand(Command); - DAC960_UserCritical("Kill of Physical Device %d:%d %s\n", - Controller, Channel, TargetID, - (Command->V2.CommandStatus - == DAC960_V2_NormalCompletion - ? "Succeeded" : "Failed")); - } - else if (strncmp(UserCommand, "make-online", 11) == 0 && - DAC960_ParsePhysicalDevice(Controller, &UserCommand[11], - &Channel, &TargetID) && - DAC960_V2_TranslatePhysicalDevice(Command, Channel, TargetID, - &LogicalDeviceNumber)) - { - CommandMailbox->SetDeviceState.LogicalDevice.LogicalDeviceNumber = - LogicalDeviceNumber; - CommandMailbox->SetDeviceState.IOCTL_Opcode = - DAC960_V2_SetDeviceState; - CommandMailbox->SetDeviceState.DeviceState.PhysicalDeviceState = - DAC960_V2_Device_Online; - DAC960_ExecuteCommand(Command); - DAC960_UserCritical("Make Online of Physical Device %d:%d %s\n", - Controller, Channel, TargetID, - (Command->V2.CommandStatus - == DAC960_V2_NormalCompletion - ? "Succeeded" : "Failed")); - } - else if (strncmp(UserCommand, "make-standby", 12) == 0 && - DAC960_ParsePhysicalDevice(Controller, &UserCommand[12], - &Channel, &TargetID) && - DAC960_V2_TranslatePhysicalDevice(Command, Channel, TargetID, - &LogicalDeviceNumber)) - { - CommandMailbox->SetDeviceState.LogicalDevice.LogicalDeviceNumber = - LogicalDeviceNumber; - CommandMailbox->SetDeviceState.IOCTL_Opcode = - DAC960_V2_SetDeviceState; - CommandMailbox->SetDeviceState.DeviceState.PhysicalDeviceState = - DAC960_V2_Device_Standby; - DAC960_ExecuteCommand(Command); - DAC960_UserCritical("Make Standby of Physical Device %d:%d %s\n", - Controller, Channel, TargetID, - (Command->V2.CommandStatus - == DAC960_V2_NormalCompletion - ? "Succeeded" : "Failed")); - } - else if (strncmp(UserCommand, "rebuild", 7) == 0 && - DAC960_ParsePhysicalDevice(Controller, &UserCommand[7], - &Channel, &TargetID) && - DAC960_V2_TranslatePhysicalDevice(Command, Channel, TargetID, - &LogicalDeviceNumber)) - { - CommandMailbox->LogicalDeviceInfo.LogicalDevice.LogicalDeviceNumber = - LogicalDeviceNumber; - CommandMailbox->LogicalDeviceInfo.IOCTL_Opcode = - DAC960_V2_RebuildDeviceStart; - DAC960_ExecuteCommand(Command); - DAC960_UserCritical("Rebuild of Physical Device %d:%d %s\n", - Controller, Channel, TargetID, - (Command->V2.CommandStatus - == DAC960_V2_NormalCompletion - ? "Initiated" : "Not Initiated")); - } - else if (strncmp(UserCommand, "cancel-rebuild", 14) == 0 && - DAC960_ParsePhysicalDevice(Controller, &UserCommand[14], - &Channel, &TargetID) && - DAC960_V2_TranslatePhysicalDevice(Command, Channel, TargetID, - &LogicalDeviceNumber)) - { - CommandMailbox->LogicalDeviceInfo.LogicalDevice.LogicalDeviceNumber = - LogicalDeviceNumber; - CommandMailbox->LogicalDeviceInfo.IOCTL_Opcode = - DAC960_V2_RebuildDeviceStop; - DAC960_ExecuteCommand(Command); - DAC960_UserCritical("Rebuild of Physical Device %d:%d %s\n", - Controller, Channel, TargetID, - (Command->V2.CommandStatus - == DAC960_V2_NormalCompletion - ? "Cancelled" : "Not Cancelled")); - } - else if (strncmp(UserCommand, "check-consistency", 17) == 0 && - DAC960_ParseLogicalDrive(Controller, &UserCommand[17], - &LogicalDriveNumber)) - { - CommandMailbox->ConsistencyCheck.LogicalDevice.LogicalDeviceNumber = - LogicalDriveNumber; - CommandMailbox->ConsistencyCheck.IOCTL_Opcode = - DAC960_V2_ConsistencyCheckStart; - CommandMailbox->ConsistencyCheck.RestoreConsistency = true; - CommandMailbox->ConsistencyCheck.InitializedAreaOnly = false; - DAC960_ExecuteCommand(Command); - DAC960_UserCritical("Consistency Check of Logical Drive %d " - "(/dev/rd/c%dd%d) %s\n", - Controller, LogicalDriveNumber, - Controller->ControllerNumber, - LogicalDriveNumber, - (Command->V2.CommandStatus - == DAC960_V2_NormalCompletion - ? "Initiated" : "Not Initiated")); - } - else if (strncmp(UserCommand, "cancel-consistency-check", 24) == 0 && - DAC960_ParseLogicalDrive(Controller, &UserCommand[24], - &LogicalDriveNumber)) - { - CommandMailbox->ConsistencyCheck.LogicalDevice.LogicalDeviceNumber = - LogicalDriveNumber; - CommandMailbox->ConsistencyCheck.IOCTL_Opcode = - DAC960_V2_ConsistencyCheckStop; - DAC960_ExecuteCommand(Command); - DAC960_UserCritical("Consistency Check of Logical Drive %d " - "(/dev/rd/c%dd%d) %s\n", - Controller, LogicalDriveNumber, - Controller->ControllerNumber, - LogicalDriveNumber, - (Command->V2.CommandStatus - == DAC960_V2_NormalCompletion - ? "Cancelled" : "Not Cancelled")); - } - else if (strcmp(UserCommand, "perform-discovery") == 0) - { - CommandMailbox->Common.IOCTL_Opcode = DAC960_V2_StartDiscovery; - DAC960_ExecuteCommand(Command); - DAC960_UserCritical("Discovery %s\n", Controller, - (Command->V2.CommandStatus - == DAC960_V2_NormalCompletion - ? "Initiated" : "Not Initiated")); - if (Command->V2.CommandStatus == DAC960_V2_NormalCompletion) - { - CommandMailbox->ControllerInfo.CommandOpcode = DAC960_V2_IOCTL; - CommandMailbox->ControllerInfo.CommandControlBits - .DataTransferControllerToHost = true; - CommandMailbox->ControllerInfo.CommandControlBits - .NoAutoRequestSense = true; - CommandMailbox->ControllerInfo.DataTransferSize = - sizeof(DAC960_V2_ControllerInfo_T); - CommandMailbox->ControllerInfo.ControllerNumber = 0; - CommandMailbox->ControllerInfo.IOCTL_Opcode = - DAC960_V2_GetControllerInfo; - /* - * How does this NOT race with the queued Monitoring - * usage of this structure? - */ - CommandMailbox->ControllerInfo.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentDataPointer = - Controller->V2.NewControllerInformationDMA; - CommandMailbox->ControllerInfo.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentByteCount = - CommandMailbox->ControllerInfo.DataTransferSize; - while (1) { - DAC960_ExecuteCommand(Command); - if (!Controller->V2.NewControllerInformation->PhysicalScanActive) - break; - msleep(1000); - } - DAC960_UserCritical("Discovery Completed\n", Controller); - } - } - else if (strcmp(UserCommand, "suppress-enclosure-messages") == 0) - Controller->SuppressEnclosureMessages = true; - else DAC960_UserCritical("Illegal User Command: '%s'\n", - Controller, UserCommand); - - spin_lock_irqsave(&Controller->queue_lock, flags); - DAC960_DeallocateCommand(Command); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - return true; -} - -static int __maybe_unused dac960_proc_show(struct seq_file *m, void *v) -{ - unsigned char *StatusMessage = "OK\n"; - int ControllerNumber; - for (ControllerNumber = 0; - ControllerNumber < DAC960_ControllerCount; - ControllerNumber++) - { - DAC960_Controller_T *Controller = DAC960_Controllers[ControllerNumber]; - if (Controller == NULL) continue; - if (Controller->MonitoringAlertMode) - { - StatusMessage = "ALERT\n"; - break; - } - } - seq_puts(m, StatusMessage); - return 0; -} - -static int __maybe_unused dac960_initial_status_proc_show(struct seq_file *m, - void *v) -{ - DAC960_Controller_T *Controller = (DAC960_Controller_T *)m->private; - seq_printf(m, "%.*s", Controller->InitialStatusLength, Controller->CombinedStatusBuffer); - return 0; -} - -static int __maybe_unused dac960_current_status_proc_show(struct seq_file *m, - void *v) -{ - DAC960_Controller_T *Controller = (DAC960_Controller_T *) m->private; - unsigned char *StatusMessage = - "No Rebuild or Consistency Check in Progress\n"; - int ProgressMessageLength = strlen(StatusMessage); - if (jiffies != Controller->LastCurrentStatusTime) - { - Controller->CurrentStatusLength = 0; - DAC960_AnnounceDriver(Controller); - DAC960_ReportControllerConfiguration(Controller); - DAC960_ReportDeviceConfiguration(Controller); - if (Controller->ProgressBufferLength > 0) - ProgressMessageLength = Controller->ProgressBufferLength; - if (DAC960_CheckStatusBuffer(Controller, 2 + ProgressMessageLength)) - { - unsigned char *CurrentStatusBuffer = Controller->CurrentStatusBuffer; - CurrentStatusBuffer[Controller->CurrentStatusLength++] = ' '; - CurrentStatusBuffer[Controller->CurrentStatusLength++] = ' '; - if (Controller->ProgressBufferLength > 0) - strcpy(&CurrentStatusBuffer[Controller->CurrentStatusLength], - Controller->ProgressBuffer); - else - strcpy(&CurrentStatusBuffer[Controller->CurrentStatusLength], - StatusMessage); - Controller->CurrentStatusLength += ProgressMessageLength; - } - Controller->LastCurrentStatusTime = jiffies; - } - seq_printf(m, "%.*s", Controller->CurrentStatusLength, Controller->CurrentStatusBuffer); - return 0; -} - -static int dac960_user_command_proc_show(struct seq_file *m, void *v) -{ - DAC960_Controller_T *Controller = (DAC960_Controller_T *)m->private; - - seq_printf(m, "%.*s", Controller->UserStatusLength, Controller->UserStatusBuffer); - return 0; -} - -static int dac960_user_command_proc_open(struct inode *inode, struct file *file) -{ - return single_open(file, dac960_user_command_proc_show, PDE_DATA(inode)); -} - -static ssize_t dac960_user_command_proc_write(struct file *file, - const char __user *Buffer, - size_t Count, loff_t *pos) -{ - DAC960_Controller_T *Controller = PDE_DATA(file_inode(file)); - unsigned char CommandBuffer[80]; - int Length; - if (Count > sizeof(CommandBuffer)-1) return -EINVAL; - if (copy_from_user(CommandBuffer, Buffer, Count)) return -EFAULT; - CommandBuffer[Count] = '\0'; - Length = strlen(CommandBuffer); - if (Length > 0 && CommandBuffer[Length-1] == '\n') - CommandBuffer[--Length] = '\0'; - if (Controller->FirmwareType == DAC960_V1_Controller) - return (DAC960_V1_ExecuteUserCommand(Controller, CommandBuffer) - ? Count : -EBUSY); - else - return (DAC960_V2_ExecuteUserCommand(Controller, CommandBuffer) - ? Count : -EBUSY); -} - -static const struct file_operations dac960_user_command_proc_fops = { - .owner = THIS_MODULE, - .open = dac960_user_command_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = dac960_user_command_proc_write, -}; - -/* - DAC960_CreateProcEntries creates the /proc/rd/... entries for the - DAC960 Driver. -*/ - -static void DAC960_CreateProcEntries(DAC960_Controller_T *Controller) -{ - struct proc_dir_entry *ControllerProcEntry; - - if (DAC960_ProcDirectoryEntry == NULL) { - DAC960_ProcDirectoryEntry = proc_mkdir("rd", NULL); - proc_create_single("status", 0, DAC960_ProcDirectoryEntry, - dac960_proc_show); - } - - snprintf(Controller->ControllerName, sizeof(Controller->ControllerName), - "c%d", Controller->ControllerNumber); - ControllerProcEntry = proc_mkdir(Controller->ControllerName, - DAC960_ProcDirectoryEntry); - proc_create_single_data("initial_status", 0, ControllerProcEntry, - dac960_initial_status_proc_show, Controller); - proc_create_single_data("current_status", 0, ControllerProcEntry, - dac960_current_status_proc_show, Controller); - proc_create_data("user_command", 0600, ControllerProcEntry, &dac960_user_command_proc_fops, Controller); - Controller->ControllerProcEntry = ControllerProcEntry; -} - - -/* - DAC960_DestroyProcEntries destroys the /proc/rd/... entries for the - DAC960 Driver. -*/ - -static void DAC960_DestroyProcEntries(DAC960_Controller_T *Controller) -{ - if (Controller->ControllerProcEntry == NULL) - return; - remove_proc_entry("initial_status", Controller->ControllerProcEntry); - remove_proc_entry("current_status", Controller->ControllerProcEntry); - remove_proc_entry("user_command", Controller->ControllerProcEntry); - remove_proc_entry(Controller->ControllerName, DAC960_ProcDirectoryEntry); - Controller->ControllerProcEntry = NULL; -} - -#ifdef DAC960_GAM_MINOR - -static long DAC960_gam_get_controller_info(DAC960_ControllerInfo_T __user *UserSpaceControllerInfo) -{ - DAC960_ControllerInfo_T ControllerInfo; - DAC960_Controller_T *Controller; - int ControllerNumber; - long ErrorCode; - - if (UserSpaceControllerInfo == NULL) - ErrorCode = -EINVAL; - else ErrorCode = get_user(ControllerNumber, - &UserSpaceControllerInfo->ControllerNumber); - if (ErrorCode != 0) - goto out; - ErrorCode = -ENXIO; - if (ControllerNumber < 0 || - ControllerNumber > DAC960_ControllerCount - 1) { - goto out; - } - Controller = DAC960_Controllers[ControllerNumber]; - if (Controller == NULL) - goto out; - memset(&ControllerInfo, 0, sizeof(DAC960_ControllerInfo_T)); - ControllerInfo.ControllerNumber = ControllerNumber; - ControllerInfo.FirmwareType = Controller->FirmwareType; - ControllerInfo.Channels = Controller->Channels; - ControllerInfo.Targets = Controller->Targets; - ControllerInfo.PCI_Bus = Controller->Bus; - ControllerInfo.PCI_Device = Controller->Device; - ControllerInfo.PCI_Function = Controller->Function; - ControllerInfo.IRQ_Channel = Controller->IRQ_Channel; - ControllerInfo.PCI_Address = Controller->PCI_Address; - strcpy(ControllerInfo.ModelName, Controller->ModelName); - strcpy(ControllerInfo.FirmwareVersion, Controller->FirmwareVersion); - ErrorCode = (copy_to_user(UserSpaceControllerInfo, &ControllerInfo, - sizeof(DAC960_ControllerInfo_T)) ? -EFAULT : 0); -out: - return ErrorCode; -} - -static long DAC960_gam_v1_execute_command(DAC960_V1_UserCommand_T __user *UserSpaceUserCommand) -{ - DAC960_V1_UserCommand_T UserCommand; - DAC960_Controller_T *Controller; - DAC960_Command_T *Command = NULL; - DAC960_V1_CommandOpcode_T CommandOpcode; - DAC960_V1_CommandStatus_T CommandStatus; - DAC960_V1_DCDB_T DCDB; - DAC960_V1_DCDB_T *DCDB_IOBUF = NULL; - dma_addr_t DCDB_IOBUFDMA; - unsigned long flags; - int ControllerNumber, DataTransferLength; - unsigned char *DataTransferBuffer = NULL; - dma_addr_t DataTransferBufferDMA; - long ErrorCode; - - if (UserSpaceUserCommand == NULL) { - ErrorCode = -EINVAL; - goto out; - } - if (copy_from_user(&UserCommand, UserSpaceUserCommand, - sizeof(DAC960_V1_UserCommand_T))) { - ErrorCode = -EFAULT; - goto out; - } - ControllerNumber = UserCommand.ControllerNumber; - ErrorCode = -ENXIO; - if (ControllerNumber < 0 || - ControllerNumber > DAC960_ControllerCount - 1) - goto out; - Controller = DAC960_Controllers[ControllerNumber]; - if (Controller == NULL) - goto out; - ErrorCode = -EINVAL; - if (Controller->FirmwareType != DAC960_V1_Controller) - goto out; - CommandOpcode = UserCommand.CommandMailbox.Common.CommandOpcode; - DataTransferLength = UserCommand.DataTransferLength; - if (CommandOpcode & 0x80) - goto out; - if (CommandOpcode == DAC960_V1_DCDB) - { - if (copy_from_user(&DCDB, UserCommand.DCDB, - sizeof(DAC960_V1_DCDB_T))) { - ErrorCode = -EFAULT; - goto out; - } - if (DCDB.Channel >= DAC960_V1_MaxChannels) - goto out; - if (!((DataTransferLength == 0 && - DCDB.Direction - == DAC960_V1_DCDB_NoDataTransfer) || - (DataTransferLength > 0 && - DCDB.Direction - == DAC960_V1_DCDB_DataTransferDeviceToSystem) || - (DataTransferLength < 0 && - DCDB.Direction - == DAC960_V1_DCDB_DataTransferSystemToDevice))) - goto out; - if (((DCDB.TransferLengthHigh4 << 16) | DCDB.TransferLength) - != abs(DataTransferLength)) - goto out; - DCDB_IOBUF = pci_alloc_consistent(Controller->PCIDevice, - sizeof(DAC960_V1_DCDB_T), &DCDB_IOBUFDMA); - if (DCDB_IOBUF == NULL) { - ErrorCode = -ENOMEM; - goto out; - } - } - ErrorCode = -ENOMEM; - if (DataTransferLength > 0) - { - DataTransferBuffer = pci_zalloc_consistent(Controller->PCIDevice, - DataTransferLength, - &DataTransferBufferDMA); - if (DataTransferBuffer == NULL) - goto out; - } - else if (DataTransferLength < 0) - { - DataTransferBuffer = pci_alloc_consistent(Controller->PCIDevice, - -DataTransferLength, &DataTransferBufferDMA); - if (DataTransferBuffer == NULL) - goto out; - if (copy_from_user(DataTransferBuffer, - UserCommand.DataTransferBuffer, - -DataTransferLength)) { - ErrorCode = -EFAULT; - goto out; - } - } - if (CommandOpcode == DAC960_V1_DCDB) - { - spin_lock_irqsave(&Controller->queue_lock, flags); - while ((Command = DAC960_AllocateCommand(Controller)) == NULL) - DAC960_WaitForCommand(Controller); - while (Controller->V1.DirectCommandActive[DCDB.Channel] - [DCDB.TargetID]) - { - spin_unlock_irq(&Controller->queue_lock); - __wait_event(Controller->CommandWaitQueue, - !Controller->V1.DirectCommandActive - [DCDB.Channel][DCDB.TargetID]); - spin_lock_irq(&Controller->queue_lock); - } - Controller->V1.DirectCommandActive[DCDB.Channel] - [DCDB.TargetID] = true; - spin_unlock_irqrestore(&Controller->queue_lock, flags); - DAC960_V1_ClearCommand(Command); - Command->CommandType = DAC960_ImmediateCommand; - memcpy(&Command->V1.CommandMailbox, &UserCommand.CommandMailbox, - sizeof(DAC960_V1_CommandMailbox_T)); - Command->V1.CommandMailbox.Type3.BusAddress = DCDB_IOBUFDMA; - DCDB.BusAddress = DataTransferBufferDMA; - memcpy(DCDB_IOBUF, &DCDB, sizeof(DAC960_V1_DCDB_T)); - } - else - { - spin_lock_irqsave(&Controller->queue_lock, flags); - while ((Command = DAC960_AllocateCommand(Controller)) == NULL) - DAC960_WaitForCommand(Controller); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - DAC960_V1_ClearCommand(Command); - Command->CommandType = DAC960_ImmediateCommand; - memcpy(&Command->V1.CommandMailbox, &UserCommand.CommandMailbox, - sizeof(DAC960_V1_CommandMailbox_T)); - if (DataTransferBuffer != NULL) - Command->V1.CommandMailbox.Type3.BusAddress = - DataTransferBufferDMA; - } - DAC960_ExecuteCommand(Command); - CommandStatus = Command->V1.CommandStatus; - spin_lock_irqsave(&Controller->queue_lock, flags); - DAC960_DeallocateCommand(Command); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - if (DataTransferLength > 0) - { - if (copy_to_user(UserCommand.DataTransferBuffer, - DataTransferBuffer, DataTransferLength)) { - ErrorCode = -EFAULT; - goto Failure1; - } - } - if (CommandOpcode == DAC960_V1_DCDB) - { - /* - I don't believe Target or Channel in the DCDB_IOBUF - should be any different from the contents of DCDB. - */ - Controller->V1.DirectCommandActive[DCDB.Channel] - [DCDB.TargetID] = false; - if (copy_to_user(UserCommand.DCDB, DCDB_IOBUF, - sizeof(DAC960_V1_DCDB_T))) { - ErrorCode = -EFAULT; - goto Failure1; - } - } - ErrorCode = CommandStatus; - Failure1: - if (DataTransferBuffer != NULL) - pci_free_consistent(Controller->PCIDevice, abs(DataTransferLength), - DataTransferBuffer, DataTransferBufferDMA); - if (DCDB_IOBUF != NULL) - pci_free_consistent(Controller->PCIDevice, sizeof(DAC960_V1_DCDB_T), - DCDB_IOBUF, DCDB_IOBUFDMA); - out: - return ErrorCode; -} - -static long DAC960_gam_v2_execute_command(DAC960_V2_UserCommand_T __user *UserSpaceUserCommand) -{ - DAC960_V2_UserCommand_T UserCommand; - DAC960_Controller_T *Controller; - DAC960_Command_T *Command = NULL; - DAC960_V2_CommandMailbox_T *CommandMailbox; - DAC960_V2_CommandStatus_T CommandStatus; - unsigned long flags; - int ControllerNumber, DataTransferLength; - int DataTransferResidue, RequestSenseLength; - unsigned char *DataTransferBuffer = NULL; - dma_addr_t DataTransferBufferDMA; - unsigned char *RequestSenseBuffer = NULL; - dma_addr_t RequestSenseBufferDMA; - long ErrorCode = -EINVAL; - - if (UserSpaceUserCommand == NULL) - goto out; - if (copy_from_user(&UserCommand, UserSpaceUserCommand, - sizeof(DAC960_V2_UserCommand_T))) { - ErrorCode = -EFAULT; - goto out; - } - ErrorCode = -ENXIO; - ControllerNumber = UserCommand.ControllerNumber; - if (ControllerNumber < 0 || - ControllerNumber > DAC960_ControllerCount - 1) - goto out; - Controller = DAC960_Controllers[ControllerNumber]; - if (Controller == NULL) - goto out; - if (Controller->FirmwareType != DAC960_V2_Controller){ - ErrorCode = -EINVAL; - goto out; - } - DataTransferLength = UserCommand.DataTransferLength; - ErrorCode = -ENOMEM; - if (DataTransferLength > 0) - { - DataTransferBuffer = pci_zalloc_consistent(Controller->PCIDevice, - DataTransferLength, - &DataTransferBufferDMA); - if (DataTransferBuffer == NULL) - goto out; - } - else if (DataTransferLength < 0) - { - DataTransferBuffer = pci_alloc_consistent(Controller->PCIDevice, - -DataTransferLength, &DataTransferBufferDMA); - if (DataTransferBuffer == NULL) - goto out; - if (copy_from_user(DataTransferBuffer, - UserCommand.DataTransferBuffer, - -DataTransferLength)) { - ErrorCode = -EFAULT; - goto Failure2; - } - } - RequestSenseLength = UserCommand.RequestSenseLength; - if (RequestSenseLength > 0) - { - RequestSenseBuffer = pci_zalloc_consistent(Controller->PCIDevice, - RequestSenseLength, - &RequestSenseBufferDMA); - if (RequestSenseBuffer == NULL) - { - ErrorCode = -ENOMEM; - goto Failure2; - } - } - spin_lock_irqsave(&Controller->queue_lock, flags); - while ((Command = DAC960_AllocateCommand(Controller)) == NULL) - DAC960_WaitForCommand(Controller); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - DAC960_V2_ClearCommand(Command); - Command->CommandType = DAC960_ImmediateCommand; - CommandMailbox = &Command->V2.CommandMailbox; - memcpy(CommandMailbox, &UserCommand.CommandMailbox, - sizeof(DAC960_V2_CommandMailbox_T)); - CommandMailbox->Common.CommandControlBits - .AdditionalScatterGatherListMemory = false; - CommandMailbox->Common.CommandControlBits - .NoAutoRequestSense = true; - CommandMailbox->Common.DataTransferSize = 0; - CommandMailbox->Common.DataTransferPageNumber = 0; - memset(&CommandMailbox->Common.DataTransferMemoryAddress, 0, - sizeof(DAC960_V2_DataTransferMemoryAddress_T)); - if (DataTransferLength != 0) - { - if (DataTransferLength > 0) - { - CommandMailbox->Common.CommandControlBits - .DataTransferControllerToHost = true; - CommandMailbox->Common.DataTransferSize = DataTransferLength; - } - else - { - CommandMailbox->Common.CommandControlBits - .DataTransferControllerToHost = false; - CommandMailbox->Common.DataTransferSize = -DataTransferLength; - } - CommandMailbox->Common.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentDataPointer = DataTransferBufferDMA; - CommandMailbox->Common.DataTransferMemoryAddress - .ScatterGatherSegments[0] - .SegmentByteCount = - CommandMailbox->Common.DataTransferSize; - } - if (RequestSenseLength > 0) - { - CommandMailbox->Common.CommandControlBits - .NoAutoRequestSense = false; - CommandMailbox->Common.RequestSenseSize = RequestSenseLength; - CommandMailbox->Common.RequestSenseBusAddress = - RequestSenseBufferDMA; - } - DAC960_ExecuteCommand(Command); - CommandStatus = Command->V2.CommandStatus; - RequestSenseLength = Command->V2.RequestSenseLength; - DataTransferResidue = Command->V2.DataTransferResidue; - spin_lock_irqsave(&Controller->queue_lock, flags); - DAC960_DeallocateCommand(Command); - spin_unlock_irqrestore(&Controller->queue_lock, flags); - if (RequestSenseLength > UserCommand.RequestSenseLength) - RequestSenseLength = UserCommand.RequestSenseLength; - if (copy_to_user(&UserSpaceUserCommand->DataTransferLength, - &DataTransferResidue, - sizeof(DataTransferResidue))) { - ErrorCode = -EFAULT; - goto Failure2; - } - if (copy_to_user(&UserSpaceUserCommand->RequestSenseLength, - &RequestSenseLength, sizeof(RequestSenseLength))) { - ErrorCode = -EFAULT; - goto Failure2; - } - if (DataTransferLength > 0) - { - if (copy_to_user(UserCommand.DataTransferBuffer, - DataTransferBuffer, DataTransferLength)) { - ErrorCode = -EFAULT; - goto Failure2; - } - } - if (RequestSenseLength > 0) - { - if (copy_to_user(UserCommand.RequestSenseBuffer, - RequestSenseBuffer, RequestSenseLength)) { - ErrorCode = -EFAULT; - goto Failure2; - } - } - ErrorCode = CommandStatus; - Failure2: - pci_free_consistent(Controller->PCIDevice, abs(DataTransferLength), - DataTransferBuffer, DataTransferBufferDMA); - if (RequestSenseBuffer != NULL) - pci_free_consistent(Controller->PCIDevice, RequestSenseLength, - RequestSenseBuffer, RequestSenseBufferDMA); -out: - return ErrorCode; -} - -static long DAC960_gam_v2_get_health_status(DAC960_V2_GetHealthStatus_T __user *UserSpaceGetHealthStatus) -{ - DAC960_V2_GetHealthStatus_T GetHealthStatus; - DAC960_V2_HealthStatusBuffer_T HealthStatusBuffer; - DAC960_Controller_T *Controller; - int ControllerNumber; - long ErrorCode; - - if (UserSpaceGetHealthStatus == NULL) { - ErrorCode = -EINVAL; - goto out; - } - if (copy_from_user(&GetHealthStatus, UserSpaceGetHealthStatus, - sizeof(DAC960_V2_GetHealthStatus_T))) { - ErrorCode = -EFAULT; - goto out; - } - ErrorCode = -ENXIO; - ControllerNumber = GetHealthStatus.ControllerNumber; - if (ControllerNumber < 0 || - ControllerNumber > DAC960_ControllerCount - 1) - goto out; - Controller = DAC960_Controllers[ControllerNumber]; - if (Controller == NULL) - goto out; - if (Controller->FirmwareType != DAC960_V2_Controller) { - ErrorCode = -EINVAL; - goto out; - } - if (copy_from_user(&HealthStatusBuffer, - GetHealthStatus.HealthStatusBuffer, - sizeof(DAC960_V2_HealthStatusBuffer_T))) { - ErrorCode = -EFAULT; - goto out; - } - ErrorCode = wait_event_interruptible_timeout(Controller->HealthStatusWaitQueue, - !(Controller->V2.HealthStatusBuffer->StatusChangeCounter - == HealthStatusBuffer.StatusChangeCounter && - Controller->V2.HealthStatusBuffer->NextEventSequenceNumber - == HealthStatusBuffer.NextEventSequenceNumber), - DAC960_MonitoringTimerInterval); - if (ErrorCode == -ERESTARTSYS) { - ErrorCode = -EINTR; - goto out; - } - if (copy_to_user(GetHealthStatus.HealthStatusBuffer, - Controller->V2.HealthStatusBuffer, - sizeof(DAC960_V2_HealthStatusBuffer_T))) - ErrorCode = -EFAULT; - else - ErrorCode = 0; - -out: - return ErrorCode; -} - -/* - * DAC960_gam_ioctl is the ioctl function for performing RAID operations. -*/ - -static long DAC960_gam_ioctl(struct file *file, unsigned int Request, - unsigned long Argument) -{ - long ErrorCode = 0; - void __user *argp = (void __user *)Argument; - if (!capable(CAP_SYS_ADMIN)) return -EACCES; - - mutex_lock(&DAC960_mutex); - switch (Request) - { - case DAC960_IOCTL_GET_CONTROLLER_COUNT: - ErrorCode = DAC960_ControllerCount; - break; - case DAC960_IOCTL_GET_CONTROLLER_INFO: - ErrorCode = DAC960_gam_get_controller_info(argp); - break; - case DAC960_IOCTL_V1_EXECUTE_COMMAND: - ErrorCode = DAC960_gam_v1_execute_command(argp); - break; - case DAC960_IOCTL_V2_EXECUTE_COMMAND: - ErrorCode = DAC960_gam_v2_execute_command(argp); - break; - case DAC960_IOCTL_V2_GET_HEALTH_STATUS: - ErrorCode = DAC960_gam_v2_get_health_status(argp); - break; - default: - ErrorCode = -ENOTTY; - } - mutex_unlock(&DAC960_mutex); - return ErrorCode; -} - -static const struct file_operations DAC960_gam_fops = { - .owner = THIS_MODULE, - .unlocked_ioctl = DAC960_gam_ioctl, - .llseek = noop_llseek, -}; - -static struct miscdevice DAC960_gam_dev = { - DAC960_GAM_MINOR, - "dac960_gam", - &DAC960_gam_fops -}; - -static int DAC960_gam_init(void) -{ - int ret; - - ret = misc_register(&DAC960_gam_dev); - if (ret) - printk(KERN_ERR "DAC960_gam: can't misc_register on minor %d\n", DAC960_GAM_MINOR); - return ret; -} - -static void DAC960_gam_cleanup(void) -{ - misc_deregister(&DAC960_gam_dev); -} - -#endif /* DAC960_GAM_MINOR */ - -static struct DAC960_privdata DAC960_GEM_privdata = { - .HardwareType = DAC960_GEM_Controller, - .FirmwareType = DAC960_V2_Controller, - .InterruptHandler = DAC960_GEM_InterruptHandler, - .MemoryWindowSize = DAC960_GEM_RegisterWindowSize, -}; - - -static struct DAC960_privdata DAC960_BA_privdata = { - .HardwareType = DAC960_BA_Controller, - .FirmwareType = DAC960_V2_Controller, - .InterruptHandler = DAC960_BA_InterruptHandler, - .MemoryWindowSize = DAC960_BA_RegisterWindowSize, -}; - -static struct DAC960_privdata DAC960_LP_privdata = { - .HardwareType = DAC960_LP_Controller, - .FirmwareType = DAC960_V2_Controller, - .InterruptHandler = DAC960_LP_InterruptHandler, - .MemoryWindowSize = DAC960_LP_RegisterWindowSize, -}; - -static struct DAC960_privdata DAC960_LA_privdata = { - .HardwareType = DAC960_LA_Controller, - .FirmwareType = DAC960_V1_Controller, - .InterruptHandler = DAC960_LA_InterruptHandler, - .MemoryWindowSize = DAC960_LA_RegisterWindowSize, -}; - -static struct DAC960_privdata DAC960_PG_privdata = { - .HardwareType = DAC960_PG_Controller, - .FirmwareType = DAC960_V1_Controller, - .InterruptHandler = DAC960_PG_InterruptHandler, - .MemoryWindowSize = DAC960_PG_RegisterWindowSize, -}; - -static struct DAC960_privdata DAC960_PD_privdata = { - .HardwareType = DAC960_PD_Controller, - .FirmwareType = DAC960_V1_Controller, - .InterruptHandler = DAC960_PD_InterruptHandler, - .MemoryWindowSize = DAC960_PD_RegisterWindowSize, -}; - -static struct DAC960_privdata DAC960_P_privdata = { - .HardwareType = DAC960_P_Controller, - .FirmwareType = DAC960_V1_Controller, - .InterruptHandler = DAC960_P_InterruptHandler, - .MemoryWindowSize = DAC960_PD_RegisterWindowSize, -}; - -static const struct pci_device_id DAC960_id_table[] = { - { - .vendor = PCI_VENDOR_ID_MYLEX, - .device = PCI_DEVICE_ID_MYLEX_DAC960_GEM, - .subvendor = PCI_VENDOR_ID_MYLEX, - .subdevice = PCI_ANY_ID, - .driver_data = (unsigned long) &DAC960_GEM_privdata, - }, - { - .vendor = PCI_VENDOR_ID_MYLEX, - .device = PCI_DEVICE_ID_MYLEX_DAC960_BA, - .subvendor = PCI_ANY_ID, - .subdevice = PCI_ANY_ID, - .driver_data = (unsigned long) &DAC960_BA_privdata, - }, - { - .vendor = PCI_VENDOR_ID_MYLEX, - .device = PCI_DEVICE_ID_MYLEX_DAC960_LP, - .subvendor = PCI_ANY_ID, - .subdevice = PCI_ANY_ID, - .driver_data = (unsigned long) &DAC960_LP_privdata, - }, - { - .vendor = PCI_VENDOR_ID_DEC, - .device = PCI_DEVICE_ID_DEC_21285, - .subvendor = PCI_VENDOR_ID_MYLEX, - .subdevice = PCI_DEVICE_ID_MYLEX_DAC960_LA, - .driver_data = (unsigned long) &DAC960_LA_privdata, - }, - { - .vendor = PCI_VENDOR_ID_MYLEX, - .device = PCI_DEVICE_ID_MYLEX_DAC960_PG, - .subvendor = PCI_ANY_ID, - .subdevice = PCI_ANY_ID, - .driver_data = (unsigned long) &DAC960_PG_privdata, - }, - { - .vendor = PCI_VENDOR_ID_MYLEX, - .device = PCI_DEVICE_ID_MYLEX_DAC960_PD, - .subvendor = PCI_ANY_ID, - .subdevice = PCI_ANY_ID, - .driver_data = (unsigned long) &DAC960_PD_privdata, - }, - { - .vendor = PCI_VENDOR_ID_MYLEX, - .device = PCI_DEVICE_ID_MYLEX_DAC960_P, - .subvendor = PCI_ANY_ID, - .subdevice = PCI_ANY_ID, - .driver_data = (unsigned long) &DAC960_P_privdata, - }, - {0, }, -}; - -MODULE_DEVICE_TABLE(pci, DAC960_id_table); - -static struct pci_driver DAC960_pci_driver = { - .name = "DAC960", - .id_table = DAC960_id_table, - .probe = DAC960_Probe, - .remove = DAC960_Remove, -}; - -static int __init DAC960_init_module(void) -{ - int ret; - - ret = pci_register_driver(&DAC960_pci_driver); -#ifdef DAC960_GAM_MINOR - if (!ret) - DAC960_gam_init(); -#endif - return ret; -} - -static void __exit DAC960_cleanup_module(void) -{ - int i; - -#ifdef DAC960_GAM_MINOR - DAC960_gam_cleanup(); -#endif - - for (i = 0; i < DAC960_ControllerCount; i++) { - DAC960_Controller_T *Controller = DAC960_Controllers[i]; - if (Controller == NULL) - continue; - DAC960_FinalizeController(Controller); - } - if (DAC960_ProcDirectoryEntry != NULL) { - remove_proc_entry("rd/status", NULL); - remove_proc_entry("rd", NULL); - } - DAC960_ControllerCount = 0; - pci_unregister_driver(&DAC960_pci_driver); -} - -module_init(DAC960_init_module); -module_exit(DAC960_cleanup_module); - -MODULE_LICENSE("GPL"); diff --git a/drivers/block/DAC960.h b/drivers/block/DAC960.h deleted file mode 100644 index 1439e651928b..000000000000 --- a/drivers/block/DAC960.h +++ /dev/null @@ -1,4414 +0,0 @@ -/* - - Linux Driver for Mylex DAC960/AcceleRAID/eXtremeRAID PCI RAID Controllers - - Copyright 1998-2001 by Leonard N. Zubkoff <lnz@dandelion.com> - - This program is free software; you may redistribute and/or modify it under - the terms of the GNU General Public License Version 2 as published by the - Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY, without even the implied warranty of MERCHANTABILITY - or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for complete details. - - The author respectfully requests that any modifications to this software be - sent directly to him for evaluation and testing. - -*/ - - -/* - Define the maximum number of DAC960 Controllers supported by this driver. -*/ - -#define DAC960_MaxControllers 8 - - -/* - Define the maximum number of Controller Channels supported by DAC960 - V1 and V2 Firmware Controllers. -*/ - -#define DAC960_V1_MaxChannels 3 -#define DAC960_V2_MaxChannels 4 - - -/* - Define the maximum number of Targets per Channel supported by DAC960 - V1 and V2 Firmware Controllers. -*/ - -#define DAC960_V1_MaxTargets 16 -#define DAC960_V2_MaxTargets 128 - - -/* - Define the maximum number of Logical Drives supported by DAC960 - V1 and V2 Firmware Controllers. -*/ - -#define DAC960_MaxLogicalDrives 32 - - -/* - Define the maximum number of Physical Devices supported by DAC960 - V1 and V2 Firmware Controllers. -*/ - -#define DAC960_V1_MaxPhysicalDevices 45 -#define DAC960_V2_MaxPhysicalDevices 272 - -/* - Define a 32/64 bit I/O Address data type. -*/ - -typedef unsigned long DAC960_IO_Address_T; - - -/* - Define a 32/64 bit PCI Bus Address data type. -*/ - -typedef unsigned long DAC960_PCI_Address_T; - - -/* - Define a 32 bit Bus Address data type. -*/ - -typedef unsigned int DAC960_BusAddress32_T; - - -/* - Define a 64 bit Bus Address data type. -*/ - -typedef unsigned long long DAC960_BusAddress64_T; - - -/* - Define a 32 bit Byte Count data type. -*/ - -typedef unsigned int DAC960_ByteCount32_T; - - -/* - Define a 64 bit Byte Count data type. -*/ - -typedef unsigned long long DAC960_ByteCount64_T; - - -/* - dma_loaf is used by helper routines to divide a region of - dma mapped memory into smaller pieces, where those pieces - are not of uniform size. - */ - -struct dma_loaf { - void *cpu_base; - dma_addr_t dma_base; - size_t length; - void *cpu_free; - dma_addr_t dma_free; -}; - -/* - Define the SCSI INQUIRY Standard Data structure. -*/ - -typedef struct DAC960_SCSI_Inquiry -{ - unsigned char PeripheralDeviceType:5; /* Byte 0 Bits 0-4 */ - unsigned char PeripheralQualifier:3; /* Byte 0 Bits 5-7 */ - unsigned char DeviceTypeModifier:7; /* Byte 1 Bits 0-6 */ - bool RMB:1; /* Byte 1 Bit 7 */ - unsigned char ANSI_ApprovedVersion:3; /* Byte 2 Bits 0-2 */ - unsigned char ECMA_Version:3; /* Byte 2 Bits 3-5 */ - unsigned char ISO_Version:2; /* Byte 2 Bits 6-7 */ - unsigned char ResponseDataFormat:4; /* Byte 3 Bits 0-3 */ - unsigned char :2; /* Byte 3 Bits 4-5 */ - bool TrmIOP:1; /* Byte 3 Bit 6 */ - bool AENC:1; /* Byte 3 Bit 7 */ - unsigned char AdditionalLength; /* Byte 4 */ - unsigned char :8; /* Byte 5 */ - unsigned char :8; /* Byte 6 */ - bool SftRe:1; /* Byte 7 Bit 0 */ - bool CmdQue:1; /* Byte 7 Bit 1 */ - bool :1; /* Byte 7 Bit 2 */ - bool Linked:1; /* Byte 7 Bit 3 */ - bool Sync:1; /* Byte 7 Bit 4 */ - bool WBus16:1; /* Byte 7 Bit 5 */ - bool WBus32:1; /* Byte 7 Bit 6 */ - bool RelAdr:1; /* Byte 7 Bit 7 */ - unsigned char VendorIdentification[8]; /* Bytes 8-15 */ - unsigned char ProductIdentification[16]; /* Bytes 16-31 */ - unsigned char ProductRevisionLevel[4]; /* Bytes 32-35 */ -} -DAC960_SCSI_Inquiry_T; - - -/* - Define the SCSI INQUIRY Unit Serial Number structure. -*/ - -typedef struct DAC960_SCSI_Inquiry_UnitSerialNumber -{ - unsigned char PeripheralDeviceType:5; /* Byte 0 Bits 0-4 */ - unsigned char PeripheralQualifier:3; /* Byte 0 Bits 5-7 */ - unsigned char PageCode; /* Byte 1 */ - unsigned char :8; /* Byte 2 */ - unsigned char PageLength; /* Byte 3 */ - unsigned char ProductSerialNumber[28]; /* Bytes 4-31 */ -} -DAC960_SCSI_Inquiry_UnitSerialNumber_T; - - -/* - Define the SCSI REQUEST SENSE Sense Key type. -*/ - -typedef enum -{ - DAC960_SenseKey_NoSense = 0x0, - DAC960_SenseKey_RecoveredError = 0x1, - DAC960_SenseKey_NotReady = 0x2, - DAC960_SenseKey_MediumError = 0x3, - DAC960_SenseKey_HardwareError = 0x4, - DAC960_SenseKey_IllegalRequest = 0x5, - DAC960_SenseKey_UnitAttention = 0x6, - DAC960_SenseKey_DataProtect = 0x7, - DAC960_SenseKey_BlankCheck = 0x8, - DAC960_SenseKey_VendorSpecific = 0x9, - DAC960_SenseKey_CopyAborted = 0xA, - DAC960_SenseKey_AbortedCommand = 0xB, - DAC960_SenseKey_Equal = 0xC, - DAC960_SenseKey_VolumeOverflow = 0xD, - DAC960_SenseKey_Miscompare = 0xE, - DAC960_SenseKey_Reserved = 0xF -} -__attribute__ ((packed)) -DAC960_SCSI_RequestSenseKey_T; - - -/* - Define the SCSI REQUEST SENSE structure. -*/ - -typedef struct DAC960_SCSI_RequestSense -{ - unsigned char ErrorCode:7; /* Byte 0 Bits 0-6 */ - bool Valid:1; /* Byte 0 Bit 7 */ - unsigned char SegmentNumber; /* Byte 1 */ - DAC960_SCSI_RequestSenseKey_T SenseKey:4; /* Byte 2 Bits 0-3 */ - unsigned char :1; /* Byte 2 Bit 4 */ - bool ILI:1; /* Byte 2 Bit 5 */ - bool EOM:1; /* Byte 2 Bit 6 */ - bool Filemark:1; /* Byte 2 Bit 7 */ - unsigned char Information[4]; /* Bytes 3-6 */ - unsigned char AdditionalSenseLength; /* Byte 7 */ - unsigned char CommandSpecificInformation[4]; /* Bytes 8-11 */ - unsigned char AdditionalSenseCode; /* Byte 12 */ - unsigned char AdditionalSenseCodeQualifier; /* Byte 13 */ -} -DAC960_SCSI_RequestSense_T; - - -/* - Define the DAC960 V1 Firmware Command Opcodes. -*/ - -typedef enum -{ - /* I/O Commands */ - DAC960_V1_ReadExtended = 0x33, - DAC960_V1_WriteExtended = 0x34, - DAC960_V1_ReadAheadExtended = 0x35, - DAC960_V1_ReadExtendedWithScatterGather = 0xB3, - DAC960_V1_WriteExtendedWithScatterGather = 0xB4, - DAC960_V1_Read = 0x36, - DAC960_V1_ReadWithScatterGather = 0xB6, - DAC960_V1_Write = 0x37, - DAC960_V1_WriteWithScatterGather = 0xB7, - DAC960_V1_DCDB = 0x04, - DAC960_V1_DCDBWithScatterGather = 0x84, - DAC960_V1_Flush = 0x0A, - /* Controller Status Related Commands */ - DAC960_V1_Enquiry = 0x53, - DAC960_V1_Enquiry2 = 0x1C, - DAC960_V1_GetLogicalDriveElement = 0x55, - DAC960_V1_GetLogicalDriveInformation = 0x19, - DAC960_V1_IOPortRead = 0x39, - DAC960_V1_IOPortWrite = 0x3A, - DAC960_V1_GetSDStats = 0x3E, - DAC960_V1_GetPDStats = 0x3F, - DAC960_V1_PerformEventLogOperation = 0x72, - /* Device Related Commands */ - DAC960_V1_StartDevice = 0x10, - DAC960_V1_GetDeviceState = 0x50, - DAC960_V1_StopChannel = 0x13, - DAC960_V1_StartChannel = 0x12, - DAC960_V1_ResetChannel = 0x1A, - /* Commands Associated with Data Consistency and Errors */ - DAC960_V1_Rebuild = 0x09, - DAC960_V1_RebuildAsync = 0x16, - DAC960_V1_CheckConsistency = 0x0F, - DAC960_V1_CheckConsistencyAsync = 0x1E, - DAC960_V1_RebuildStat = 0x0C, - DAC960_V1_GetRebuildProgress = 0x27, - DAC960_V1_RebuildControl = 0x1F, - DAC960_V1_ReadBadBlockTable = 0x0B, - DAC960_V1_ReadBadDataTable = 0x25, - DAC960_V1_ClearBadDataTable = 0x26, - DAC960_V1_GetErrorTable = 0x17, - DAC960_V1_AddCapacityAsync = 0x2A, - DAC960_V1_BackgroundInitializationControl = 0x2B, - /* Configuration Related Commands */ - DAC960_V1_ReadConfig2 = 0x3D, - DAC960_V1_WriteConfig2 = 0x3C, - DAC960_V1_ReadConfigurationOnDisk = 0x4A, - DAC960_V1_WriteConfigurationOnDisk = 0x4B, - DAC960_V1_ReadConfiguration = 0x4E, - DAC960_V1_ReadBackupConfiguration = 0x4D, - DAC960_V1_WriteConfiguration = 0x4F, - DAC960_V1_AddConfiguration = 0x4C, - DAC960_V1_ReadConfigurationLabel = 0x48, - DAC960_V1_WriteConfigurationLabel = 0x49, - /* Firmware Upgrade Related Commands */ - DAC960_V1_LoadImage = 0x20, - DAC960_V1_StoreImage = 0x21, - DAC960_V1_ProgramImage = 0x22, - /* Diagnostic Commands */ - DAC960_V1_SetDiagnosticMode = 0x31, - DAC960_V1_RunDiagnostic = 0x32, - /* Subsystem Service Commands */ - DAC960_V1_GetSubsystemData = 0x70, - DAC960_V1_SetSubsystemParameters = 0x71, - /* Version 2.xx Firmware Commands */ - DAC960_V1_Enquiry_Old = 0x05, - DAC960_V1_GetDeviceState_Old = 0x14, - DAC960_V1_Read_Old = 0x02, - DAC960_V1_Write_Old = 0x03, - DAC960_V1_ReadWithScatterGather_Old = 0x82, - DAC960_V1_WriteWithScatterGather_Old = 0x83 -} -__attribute__ ((packed)) -DAC960_V1_CommandOpcode_T; - - -/* - Define the DAC960 V1 Firmware Command Identifier type. -*/ - -typedef unsigned char DAC960_V1_CommandIdentifier_T; - - -/* - Define the DAC960 V1 Firmware Command Status Codes. -*/ - -#define DAC960_V1_NormalCompletion 0x0000 /* Common */ -#define DAC960_V1_CheckConditionReceived 0x0002 /* Common */ -#define DAC960_V1_NoDeviceAtAddress 0x0102 /* Common */ -#define DAC960_V1_InvalidDeviceAddress 0x0105 /* Common */ -#define DAC960_V1_InvalidParameter 0x0105 /* Common */ -#define DAC960_V1_IrrecoverableDataError 0x0001 /* I/O */ -#define DAC960_V1_LogicalDriveNonexistentOrOffline 0x0002 /* I/O */ -#define DAC960_V1_AccessBeyondEndOfLogicalDrive 0x0105 /* I/O */ -#define DAC960_V1_BadDataEncountered 0x010C /* I/O */ -#define DAC960_V1_DeviceBusy 0x0008 /* DCDB */ -#define DAC960_V1_DeviceNonresponsive 0x000E /* DCDB */ -#define DAC960_V1_CommandTerminatedAbnormally 0x000F /* DCDB */ -#define DAC960_V1_UnableToStartDevice 0x0002 /* Device */ -#define DAC960_V1_InvalidChannelOrTargetOrModifier 0x0105 /* Device */ -#define DAC960_V1_ChannelBusy 0x0106 /* Device */ -#define DAC960_V1_ChannelNotStopped 0x0002 /* Device */ -#define DAC960_V1_AttemptToRebuildOnlineDrive 0x0002 /* Consistency */ -#define DAC960_V1_RebuildBadBlocksEncountered 0x0003 /* Consistency */ -#define DAC960_V1_NewDiskFailedDuringRebuild 0x0004 /* Consistency */ -#define DAC960_V1_RebuildOrCheckAlreadyInProgress 0x0106 /* Consistency */ -#define DAC960_V1_DependentDiskIsDead 0x0002 /* Consistency */ -#define DAC960_V1_InconsistentBlocksFound 0x0003 /* Consistency */ -#define DAC960_V1_InvalidOrNonredundantLogicalDrive 0x0105 /* Consistency */ -#define DAC960_V1_NoRebuildOrCheckInProgress 0x0105 /* Consistency */ -#define DAC960_V1_RebuildInProgress_DataValid 0x0000 /* Consistency */ -#define DAC960_V1_RebuildFailed_LogicalDriveFailure 0x0002 /* Consistency */ -#define DAC960_V1_RebuildFailed_BadBlocksOnOther 0x0003 /* Consistency */ -#define DAC960_V1_RebuildFailed_NewDriveFailed 0x0004 /* Consistency */ -#define DAC960_V1_RebuildSuccessful 0x0100 /* Consistency */ -#define DAC960_V1_RebuildSuccessfullyTerminated 0x0107 /* Consistency */ -#define DAC960_V1_BackgroundInitSuccessful 0x0100 /* Consistency */ -#define DAC960_V1_BackgroundInitAborted 0x0005 /* Consistency */ -#define DAC960_V1_NoBackgroundInitInProgress 0x0105 /* Consistency */ -#define DAC960_V1_AddCapacityInProgress 0x0004 /* Consistency */ -#define DAC960_V1_AddCapacityFailedOrSuspended 0x00F4 /* Consistency */ -#define DAC960_V1_Config2ChecksumError 0x0002 /* Configuration */ -#define DAC960_V1_ConfigurationSuspended 0x0106 /* Configuration */ -#define DAC960_V1_FailedToConfigureNVRAM 0x0105 /* Configuration */ -#define DAC960_V1_ConfigurationNotSavedStateChange 0x0106 /* Configuration */ -#define DAC960_V1_SubsystemNotInstalled 0x0001 /* Subsystem */ -#define DAC960_V1_SubsystemFailed 0x0002 /* Subsystem */ -#define DAC960_V1_SubsystemBusy 0x0106 /* Subsystem */ - -typedef unsigned short DAC960_V1_CommandStatus_T; - - -/* - Define the DAC960 V1 Firmware Enquiry Command reply structure. -*/ - -typedef struct DAC960_V1_Enquiry -{ - unsigned char NumberOfLogicalDrives; /* Byte 0 */ - unsigned int :24; /* Bytes 1-3 */ - unsigned int LogicalDriveSizes[32]; /* Bytes 4-131 */ - unsigned short FlashAge; /* Bytes 132-133 */ - struct { - bool DeferredWriteError:1; /* Byte 134 Bit 0 */ - bool BatteryLow:1; /* Byte 134 Bit 1 */ - unsigned char :6; /* Byte 134 Bits 2-7 */ - } StatusFlags; - unsigned char :8; /* Byte 135 */ - unsigned char MinorFirmwareVersion; /* Byte 136 */ - unsigned char MajorFirmwareVersion; /* Byte 137 */ - enum { - DAC960_V1_NoStandbyRebuildOrCheckInProgress = 0x00, - DAC960_V1_StandbyRebuildInProgress = 0x01, - DAC960_V1_BackgroundRebuildInProgress = 0x02, - DAC960_V1_BackgroundCheckInProgress = 0x03, - DAC960_V1_StandbyRebuildCompletedWithError = 0xFF, - DAC960_V1_BackgroundRebuildOrCheckFailed_DriveFailed = 0xF0, - DAC960_V1_BackgroundRebuildOrCheckFailed_LogicalDriveFailed = 0xF1, - DAC960_V1_BackgroundRebuildOrCheckFailed_OtherCauses = 0xF2, - DAC960_V1_BackgroundRebuildOrCheckSuccessfullyTerminated = 0xF3 - } __attribute__ ((packed)) RebuildFlag; /* Byte 138 */ - unsigned char MaxCommands; /* Byte 139 */ - unsigned char OfflineLogicalDriveCount; /* Byte 140 */ - unsigned char :8; /* Byte 141 */ - unsigned short EventLogSequenceNumber; /* Bytes 142-143 */ - unsigned char CriticalLogicalDriveCount; /* Byte 144 */ - unsigned int :24; /* Bytes 145-147 */ - unsigned char DeadDriveCount; /* Byte 148 */ - unsigned char :8; /* Byte 149 */ - unsigned char RebuildCount; /* Byte 150 */ - struct { - unsigned char :3; /* Byte 151 Bits 0-2 */ - bool BatteryBackupUnitPresent:1; /* Byte 151 Bit 3 */ - unsigned char :3; /* Byte 151 Bits 4-6 */ - unsigned char :1; /* Byte 151 Bit 7 */ - } MiscFlags; - struct { - unsigned char TargetID; - unsigned char Channel; - } DeadDrives[21]; /* Bytes 152-194 */ - unsigned char Reserved[62]; /* Bytes 195-255 */ -} -__attribute__ ((packed)) -DAC960_V1_Enquiry_T; - - -/* - Define the DAC960 V1 Firmware Enquiry2 Command reply structure. -*/ - -typedef struct DAC960_V1_Enquiry2 -{ - struct { - enum { - DAC960_V1_P_PD_PU = 0x01, - DAC960_V1_PL = 0x02, - DAC960_V1_PG = 0x10, - DAC960_V1_PJ = 0x11, - DAC960_V1_PR = 0x12, - DAC960_V1_PT = 0x13, - DAC960_V1_PTL0 = 0x14, - DAC960_V1_PRL = 0x15, - DAC960_V1_PTL1 = 0x16, - DAC960_V1_1164P = 0x20 - } __attribute__ ((packed)) SubModel; /* Byte 0 */ - unsigned char ActualChannels; /* Byte 1 */ - enum { - DAC960_V1_FiveChannelBoard = 0x01, - DAC960_V1_ThreeChannelBoard = 0x02, - DAC960_V1_TwoChannelBoard = 0x03, - DAC960_V1_ThreeChannelASIC_DAC = 0x04 - } __attribute__ ((packed)) Model; /* Byte 2 */ - enum { - DAC960_V1_EISA_Controller = 0x01, - DAC960_V1_MicroChannel_Controller = 0x02, - DAC960_V1_PCI_Controller = 0x03, - DAC960_V1_SCSItoSCSI_Controller = 0x08 - } __attribute__ ((packed)) ProductFamily; /* Byte 3 */ - } HardwareID; /* Bytes 0-3 */ - /* MajorVersion.MinorVersion-FirmwareType-TurnID */ - struct { - unsigned char MajorVersion; /* Byte 4 */ - unsigned char MinorVersion; /* Byte 5 */ - unsigned char TurnID; /* Byte 6 */ - char FirmwareType; /* Byte 7 */ - } FirmwareID; /* Bytes 4-7 */ - unsigned char :8; /* Byte 8 */ - unsigned int :24; /* Bytes 9-11 */ - unsigned char ConfiguredChannels; /* Byte 12 */ - unsigned char ActualChannels; /* Byte 13 */ - unsigned char MaxTargets; /* Byte 14 */ - unsigned char MaxTags; /* Byte 15 */ - unsigned char MaxLogicalDrives; /* Byte 16 */ - unsigned char MaxArms; /* Byte 17 */ - unsigned char MaxSpans; /* Byte 18 */ - unsigned char :8; /* Byte 19 */ - unsigned int :32; /* Bytes 20-23 */ - unsigned int MemorySize; /* Bytes 24-27 */ - unsigned int CacheSize; /* Bytes 28-31 */ - unsigned int FlashMemorySize; /* Bytes 32-35 */ - unsigned int NonVolatileMemorySize; /* Bytes 36-39 */ - struct { - enum { - DAC960_V1_RamType_DRAM = 0x0, - DAC960_V1_RamType_EDO = 0x1, - DAC960_V1_RamType_SDRAM = 0x2, - DAC960_V1_RamType_Last = 0x7 - } __attribute__ ((packed)) RamType:3; /* Byte 40 Bits 0-2 */ - enum { - DAC960_V1_ErrorCorrection_None = 0x0, - DAC960_V1_ErrorCorrection_Parity = 0x1, - DAC960_V1_ErrorCorrection_ECC = 0x2, - DAC960_V1_ErrorCorrection_Last = 0x7 - } __attribute__ ((packed)) ErrorCorrection:3; /* Byte 40 Bits 3-5 */ - bool FastPageMode:1; /* Byte 40 Bit 6 */ - bool LowPowerMemory:1; /* Byte 40 Bit 7 */ - unsigned char :8; /* Bytes 41 */ - } MemoryType; - unsigned short ClockSpeed; /* Bytes 42-43 */ - unsigned short MemorySpeed; /* Bytes 44-45 */ - unsigned short HardwareSpeed; /* Bytes 46-47 */ - unsigned int :32; /* Bytes 48-51 */ - unsigned int :32; /* Bytes 52-55 */ - unsigned char :8; /* Byte 56 */ - unsigned char :8; /* Byte 57 */ - unsigned short :16; /* Bytes 58-59 */ - unsigned short MaxCommands; /* Bytes 60-61 */ - unsigned short MaxScatterGatherEntries; /* Bytes 62-63 */ - unsigned short MaxDriveCommands; /* Bytes 64-65 */ - unsigned short MaxIODescriptors; /* Bytes 66-67 */ - unsigned short MaxCombinedSectors; /* Bytes 68-69 */ - unsigned char Latency; /* Byte 70 */ - unsigned char :8; /* Byte 71 */ - unsigned char SCSITimeout; /* Byte 72 */ - unsigned char :8; /* Byte 73 */ - unsigned short MinFreeLines; /* Bytes 74-75 */ - unsigned int :32; /* Bytes 76-79 */ - unsigned int :32; /* Bytes 80-83 */ - unsigned char RebuildRateConstant; /* Byte 84 */ - unsigned char :8; /* Byte 85 */ - unsigned char :8; /* Byte 86 */ - unsigned char :8; /* Byte 87 */ - unsigned int :32; /* Bytes 88-91 */ - unsigned int :32; /* Bytes 92-95 */ - unsigned short PhysicalDriveBlockSize; /* Bytes 96-97 */ - unsigned short LogicalDriveBlockSize; /* Bytes 98-99 */ - unsigned short MaxBlocksPerCommand; /* Bytes 100-101 */ - unsigned short BlockFactor; /* Bytes 102-103 */ - unsigned short CacheLineSize; /* Bytes 104-105 */ - struct { - enum { - DAC960_V1_Narrow_8bit = 0x0, - DAC960_V1_Wide_16bit = 0x1, - DAC960_V1_Wide_32bit = 0x2 - } __attribute__ ((packed)) BusWidth:2; /* Byte 106 Bits 0-1 */ - enum { - DAC960_V1_Fast = 0x0, - DAC960_V1_Ultra = 0x1, - DAC960_V1_Ultra2 = 0x2 - } __attribute__ ((packed)) BusSpeed:2; /* Byte 106 Bits 2-3 */ - bool Differential:1; /* Byte 106 Bit 4 */ - unsigned char :3; /* Byte 106 Bits 5-7 */ - } SCSICapability; - unsigned char :8; /* Byte 107 */ - unsigned int :32; /* Bytes 108-111 */ - unsigned short FirmwareBuildNumber; /* Bytes 112-113 */ - enum { - DAC960_V1_AEMI = 0x01, - DAC960_V1_OEM1 = 0x02, - DAC960_V1_OEM2 = 0x04, - DAC960_V1_OEM3 = 0x08, - DAC960_V1_Conner = 0x10, - DAC960_V1_SAFTE = 0x20 - } __attribute__ ((packed)) FaultManagementType; /* Byte 114 */ - unsigned char :8; /* Byte 115 */ - struct { - bool Clustering:1; /* Byte 116 Bit 0 */ - bool MylexOnlineRAIDExpansion:1; /* Byte 116 Bit 1 */ - bool ReadAhead:1; /* Byte 116 Bit 2 */ - bool BackgroundInitialization:1; /* Byte 116 Bit 3 */ - unsigned int :28; /* Bytes 116-119 */ - } FirmwareFeatures; - unsigned int :32; /* Bytes 120-123 */ - unsigned int :32; /* Bytes 124-127 */ -} -DAC960_V1_Enquiry2_T; - - -/* - Define the DAC960 V1 Firmware Logical Drive State type. -*/ - -typedef enum -{ - DAC960_V1_LogicalDrive_Online = 0x03, - DAC960_V1_LogicalDrive_Critical = 0x04, - DAC960_V1_LogicalDrive_Offline = 0xFF -} -__attribute__ ((packed)) -DAC960_V1_LogicalDriveState_T; - - -/* - Define the DAC960 V1 Firmware Logical Drive Information structure. -*/ - -typedef struct DAC960_V1_LogicalDriveInformation -{ - unsigned int LogicalDriveSize; /* Bytes 0-3 */ - DAC960_V1_LogicalDriveState_T LogicalDriveState; /* Byte 4 */ - unsigned char RAIDLevel:7; /* Byte 5 Bits 0-6 */ - bool WriteBack:1; /* Byte 5 Bit 7 */ - unsigned short :16; /* Bytes 6-7 */ -} -DAC960_V1_LogicalDriveInformation_T; - - -/* - Define the DAC960 V1 Firmware Get Logical Drive Information Command - reply structure. -*/ - -typedef DAC960_V1_LogicalDriveInformation_T - DAC960_V1_LogicalDriveInformationArray_T[DAC960_MaxLogicalDrives]; - - -/* - Define the DAC960 V1 Firmware Perform Event Log Operation Types. -*/ - -typedef enum -{ - DAC960_V1_GetEventLogEntry = 0x00 -} -__attribute__ ((packed)) -DAC960_V1_PerformEventLogOpType_T; - - -/* - Define the DAC960 V1 Firmware Get Event Log Entry Command reply structure. -*/ - -typedef struct DAC960_V1_EventLogEntry -{ - unsigned char MessageType; /* Byte 0 */ - unsigned char MessageLength; /* Byte 1 */ - unsigned char TargetID:5; /* Byte 2 Bits 0-4 */ - unsigned char Channel:3; /* Byte 2 Bits 5-7 */ - unsigned char LogicalUnit:6; /* Byte 3 Bits 0-5 */ - unsigned char :2; /* Byte 3 Bits 6-7 */ - unsigned short SequenceNumber; /* Bytes 4-5 */ - unsigned char ErrorCode:7; /* Byte 6 Bits 0-6 */ - bool Valid:1; /* Byte 6 Bit 7 */ - unsigned char SegmentNumber; /* Byte 7 */ - DAC960_SCSI_RequestSenseKey_T SenseKey:4; /* Byte 8 Bits 0-3 */ - unsigned char :1; /* Byte 8 Bit 4 */ - bool ILI:1; /* Byte 8 Bit 5 */ - bool EOM:1; /* Byte 8 Bit 6 */ - bool Filemark:1; /* Byte 8 Bit 7 */ - unsigned char Information[4]; /* Bytes 9-12 */ - unsigned char AdditionalSenseLength; /* Byte 13 */ - unsigned char CommandSpecificInformation[4]; /* Bytes 14-17 */ - unsigned char AdditionalSenseCode; /* Byte 18 */ - unsigned char AdditionalSenseCodeQualifier; /* Byte 19 */ - unsigned char Dummy[12]; /* Bytes 20-31 */ -} -DAC960_V1_EventLogEntry_T; - - -/* - Define the DAC960 V1 Firmware Physical Device State type. -*/ - -typedef enum -{ - DAC960_V1_Device_Dead = 0x00, - DAC960_V1_Device_WriteOnly = 0x02, - DAC960_V1_Device_Online = 0x03, - DAC960_V1_Device_Standby = 0x10 -} -__attribute__ ((packed)) -DAC960_V1_PhysicalDeviceState_T; - - -/* - Define the DAC960 V1 Firmware Get Device State Command reply structure. - The structure is padded by 2 bytes for compatibility with Version 2.xx - Firmware. -*/ - -typedef struct DAC960_V1_DeviceState -{ - bool Present:1; /* Byte 0 Bit 0 */ - unsigned char :7; /* Byte 0 Bits 1-7 */ - enum { - DAC960_V1_OtherType = 0x0, - DAC960_V1_DiskType = 0x1, - DAC960_V1_SequentialType = 0x2, - DAC960_V1_CDROM_or_WORM_Type = 0x3 - } __attribute__ ((packed)) DeviceType:2; /* Byte 1 Bits 0-1 */ - bool :1; /* Byte 1 Bit 2 */ - bool Fast20:1; /* Byte 1 Bit 3 */ - bool Sync:1; /* Byte 1 Bit 4 */ - bool Fast:1; /* Byte 1 Bit 5 */ - bool Wide:1; /* Byte 1 Bit 6 */ - bool TaggedQueuingSupported:1; /* Byte 1 Bit 7 */ - DAC960_V1_PhysicalDeviceState_T DeviceState; /* Byte 2 */ - unsigned char :8; /* Byte 3 */ - unsigned char SynchronousMultiplier; /* Byte 4 */ - unsigned char SynchronousOffset:5; /* Byte 5 Bits 0-4 */ - unsigned char :3; /* Byte 5 Bits 5-7 */ - unsigned int DiskSize __attribute__ ((packed)); /* Bytes 6-9 */ - unsigned short :16; /* Bytes 10-11 */ -} -DAC960_V1_DeviceState_T; - - -/* - Define the DAC960 V1 Firmware Get Rebuild Progress Command reply structure. -*/ - -typedef struct DAC960_V1_RebuildProgress -{ - unsigned int LogicalDriveNumber; /* Bytes 0-3 */ - unsigned int LogicalDriveSize; /* Bytes 4-7 */ - unsigned int RemainingBlocks; /* Bytes 8-11 */ -} -DAC960_V1_RebuildProgress_T; - - -/* - Define the DAC960 V1 Firmware Background Initialization Status Command - reply structure. -*/ - -typedef struct DAC960_V1_BackgroundInitializationStatus -{ - unsigned int LogicalDriveSize; /* Bytes 0-3 */ - unsigned int BlocksCompleted; /* Bytes 4-7 */ - unsigned char Reserved1[12]; /* Bytes 8-19 */ - unsigned int LogicalDriveNumber; /* Bytes 20-23 */ - unsigned char RAIDLevel; /* Byte 24 */ - enum { - DAC960_V1_BackgroundInitializationInvalid = 0x00, - DAC960_V1_BackgroundInitializationStarted = 0x02, - DAC960_V1_BackgroundInitializationInProgress = 0x04, - DAC960_V1_BackgroundInitializationSuspended = 0x05, - DAC960_V1_BackgroundInitializationCancelled = 0x06 - } __attribute__ ((packed)) Status; /* Byte 25 */ - unsigned char Reserved2[6]; /* Bytes 26-31 */ -} -DAC960_V1_BackgroundInitializationStatus_T; - - -/* - Define the DAC960 V1 Firmware Error Table Entry structure. -*/ - -typedef struct DAC960_V1_ErrorTableEntry -{ - unsigned char ParityErrorCount; /* Byte 0 */ - unsigned char SoftErrorCount; /* Byte 1 */ - unsigned char HardErrorCount; /* Byte 2 */ - unsigned char MiscErrorCount; /* Byte 3 */ -} -DAC960_V1_ErrorTableEntry_T; - - -/* - Define the DAC960 V1 Firmware Get Error Table Command reply structure. -*/ - -typedef struct DAC960_V1_ErrorTable -{ - DAC960_V1_ErrorTableEntry_T - ErrorTableEntries[DAC960_V1_MaxChannels][DAC960_V1_MaxTargets]; -} -DAC960_V1_ErrorTable_T; - - -/* - Define the DAC960 V1 Firmware Read Config2 Command reply structure. -*/ - -typedef struct DAC960_V1_Config2 -{ - unsigned char :1; /* Byte 0 Bit 0 */ - bool ActiveNegationEnabled:1; /* Byte 0 Bit 1 */ - unsigned char :5; /* Byte 0 Bits 2-6 */ - bool NoRescanIfResetReceivedDuringScan:1; /* Byte 0 Bit 7 */ - bool StorageWorksSupportEnabled:1; /* Byte 1 Bit 0 */ - bool HewlettPackardSupportEnabled:1; /* Byte 1 Bit 1 */ - bool NoDisconnectOnFirstCommand:1; /* Byte 1 Bit 2 */ - unsigned char :2; /* Byte 1 Bits 3-4 */ - bool AEMI_ARM:1; /* Byte 1 Bit 5 */ - bool AEMI_OFM:1; /* Byte 1 Bit 6 */ - unsigned char :1; /* Byte 1 Bit 7 */ - enum { - DAC960_V1_OEMID_Mylex = 0x00, - DAC960_V1_OEMID_IBM = 0x08, - DAC960_V1_OEMID_HP = 0x0A, - DAC960_V1_OEMID_DEC = 0x0C, - DAC960_V1_OEMID_Siemens = 0x10, - DAC960_V1_OEMID_Intel = 0x12 - } __attribute__ ((packed)) OEMID; /* Byte 2 */ - unsigned char OEMModelNumber; /* Byte 3 */ - unsigned char PhysicalSector; /* Byte 4 */ - unsigned char LogicalSector; /* Byte 5 */ - unsigned char BlockFactor; /* Byte 6 */ - bool ReadAheadEnabled:1; /* Byte 7 Bit 0 */ - bool LowBIOSDelay:1; /* Byte 7 Bit 1 */ - unsigned char :2; /* Byte 7 Bits 2-3 */ - bool ReassignRestrictedToOneSector:1; /* Byte 7 Bit 4 */ - unsigned char :1; /* Byte 7 Bit 5 */ - bool ForceUnitAccessDuringWriteRecovery:1; /* Byte 7 Bit 6 */ - bool EnableLeftSymmetricRAID5Algorithm:1; /* Byte 7 Bit 7 */ - unsigned char DefaultRebuildRate; /* Byte 8 */ - unsigned char :8; /* Byte 9 */ - unsigned char BlocksPerCacheLine; /* Byte 10 */ - unsigned char BlocksPerStripe; /* Byte 11 */ - struct { - enum { - DAC960_V1_Async = 0x0, - DAC960_V1_Sync_8MHz = 0x1, - DAC960_V1_Sync_5MHz = 0x2, - DAC960_V1_Sync_10or20MHz = 0x3 /* Byte 11 Bits 0-1 */ - } __attribute__ ((packed)) Speed:2; - bool Force8Bit:1; /* Byte 11 Bit 2 */ - bool DisableFast20:1; /* Byte 11 Bit 3 */ - unsigned char :3; /* Byte 11 Bits 4-6 */ - bool EnableTaggedQueuing:1; /* Byte 11 Bit 7 */ - } __attribute__ ((packed)) ChannelParameters[6]; /* Bytes 12-17 */ - unsigned char SCSIInitiatorID; /* Byte 18 */ - unsigned char :8; /* Byte 19 */ - enum { - DAC960_V1_StartupMode_ControllerSpinUp = 0x00, - DAC960_V1_StartupMode_PowerOnSpinUp = 0x01 - } __attribute__ ((packed)) StartupMode; /* Byte 20 */ - unsigned char SimultaneousDeviceSpinUpCount; /* Byte 21 */ - unsigned char SecondsDelayBetweenSpinUps; /* Byte 22 */ - unsigned char Reserved1[29]; /* Bytes 23-51 */ - bool BIOSDisabled:1; /* Byte 52 Bit 0 */ - bool CDROMBootEnabled:1; /* Byte 52 Bit 1 */ - unsigned char :3; /* Byte 52 Bits 2-4 */ - enum { - DAC960_V1_Geometry_128_32 = 0x0, - DAC960_V1_Geometry_255_63 = 0x1, - DAC960_V1_Geometry_Reserved1 = 0x2, - DAC960_V1_Geometry_Reserved2 = 0x3 - } __attribute__ ((packed)) DriveGeometry:2; /* Byte 52 Bits 5-6 */ - unsigned char :1; /* Byte 52 Bit 7 */ - unsigned char Reserved2[9]; /* Bytes 53-61 */ - unsigned short Checksum; /* Bytes 62-63 */ -} -DAC960_V1_Config2_T; - - -/* - Define the DAC960 V1 Firmware DCDB request structure. -*/ - -typedef struct DAC960_V1_DCDB -{ - unsigned char TargetID:4; /* Byte 0 Bits 0-3 */ - unsigned char Channel:4; /* Byte 0 Bits 4-7 */ - enum { - DAC960_V1_DCDB_NoDataTransfer = 0, - DAC960_V1_DCDB_DataTransferDeviceToSystem = 1, - DAC960_V1_DCDB_DataTransferSystemToDevice = 2, - DAC960_V1_DCDB_IllegalDataTransfer = 3 - } __attribute__ ((packed)) Direction:2; /* Byte 1 Bits 0-1 */ - bool EarlyStatus:1; /* Byte 1 Bit 2 */ - unsigned char :1; /* Byte 1 Bit 3 */ - enum { - DAC960_V1_DCDB_Timeout_24_hours = 0, - DAC960_V1_DCDB_Timeout_10_seconds = 1, - DAC960_V1_DCDB_Timeout_60_seconds = 2, - DAC960_V1_DCDB_Timeout_10_minutes = 3 - } __attribute__ ((packed)) Timeout:2; /* Byte 1 Bits 4-5 */ - bool NoAutomaticRequestSense:1; /* Byte 1 Bit 6 */ - bool DisconnectPermitted:1; /* Byte 1 Bit 7 */ - unsigned short TransferLength; /* Bytes 2-3 */ - DAC960_BusAddress32_T BusAddress; /* Bytes 4-7 */ - unsigned char CDBLength:4; /* Byte 8 Bits 0-3 */ - unsigned char TransferLengthHigh4:4; /* Byte 8 Bits 4-7 */ - unsigned char SenseLength; /* Byte 9 */ - unsigned char CDB[12]; /* Bytes 10-21 */ - unsigned char SenseData[64]; /* Bytes 22-85 */ - unsigned char Status; /* Byte 86 */ - unsigned char :8; /* Byte 87 */ -} -DAC960_V1_DCDB_T; - - -/* - Define the DAC960 V1 Firmware Scatter/Gather List Type 1 32 Bit Address - 32 Bit Byte Count structure. -*/ - -typedef struct DAC960_V1_ScatterGatherSegment -{ - DAC960_BusAddress32_T SegmentDataPointer; /* Bytes 0-3 */ - DAC960_ByteCount32_T SegmentByteCount; /* Bytes 4-7 */ -} -DAC960_V1_ScatterGatherSegment_T; - - -/* - Define the 13 Byte DAC960 V1 Firmware Command Mailbox structure. Bytes 13-15 - are not used. The Command Mailbox structure is padded to 16 bytes for - efficient access. -*/ - -typedef union DAC960_V1_CommandMailbox -{ - unsigned int Words[4]; /* Words 0-3 */ - unsigned char Bytes[16]; /* Bytes 0-15 */ - struct { - DAC960_V1_CommandOpcode_T CommandOpcode; /* Byte 0 */ - DAC960_V1_CommandIdentifier_T CommandIdentifier; /* Byte 1 */ - unsigned char Dummy[14]; /* Bytes 2-15 */ - } __attribute__ ((packed)) Common; - struct { - DAC960_V1_CommandOpcode_T CommandOpcode; /* Byte 0 */ - DAC960_V1_CommandIdentifier_T CommandIdentifier; /* Byte 1 */ - unsigned char Dummy1[6]; /* Bytes 2-7 */ - DAC960_BusAddress32_T BusAddress; /* Bytes 8-11 */ - unsigned char Dummy2[4]; /* Bytes 12-15 */ - } __attribute__ ((packed)) Type3; - struct { - DAC960_V1_CommandOpcode_T CommandOpcode; /* Byte 0 */ - DAC960_V1_CommandIdentifier_T CommandIdentifier; /* Byte 1 */ - unsigned char CommandOpcode2; /* Byte 2 */ - unsigned char Dummy1[5]; /* Bytes 3-7 */ - DAC960_BusAddress32_T BusAddress; /* Bytes 8-11 */ - unsigned char Dummy2[4]; /* Bytes 12-15 */ - } __attribute__ ((packed)) Type3B; - struct { - DAC960_V1_CommandOpcode_T CommandOpcode; /* Byte 0 */ - DAC960_V1_CommandIdentifier_T CommandIdentifier; /* Byte 1 */ - unsigned char Dummy1[5]; /* Bytes 2-6 */ - unsigned char LogicalDriveNumber:6; /* Byte 7 Bits 0-6 */ - bool AutoRestore:1; /* Byte 7 Bit 7 */ - unsigned char Dummy2[8]; /* Bytes 8-15 */ - } __attribute__ ((packed)) Type3C; - struct { - DAC960_V1_CommandOpcode_T CommandOpcode; /* Byte 0 */ - DAC960_V1_CommandIdentifier_T CommandIdentifier; /* Byte 1 */ - unsigned char Channel; /* Byte 2 */ - unsigned char TargetID; /* Byte 3 */ - DAC960_V1_PhysicalDeviceState_T DeviceState:5; /* Byte 4 Bits 0-4 */ - unsigned char Modifier:3; /* Byte 4 Bits 5-7 */ - unsigned char Dummy1[3]; /* Bytes 5-7 */ - DAC960_BusAddress32_T BusAddress; /* Bytes 8-11 */ - unsigned char Dummy2[4]; /* Bytes 12-15 */ - } __attribute__ ((packed)) Type3D; - struct { - DAC960_V1_CommandOpcode_T CommandOpcode; /* Byte 0 */ - DAC960_V1_CommandIdentifier_T CommandIdentifier; /* Byte 1 */ - DAC960_V1_PerformEventLogOpType_T OperationType; /* Byte 2 */ - unsigned char OperationQualifier; /* Byte 3 */ - unsigned short SequenceNumber; /* Bytes 4-5 */ - unsigned char Dummy1[2]; /* Bytes 6-7 */ - DAC960_BusAddress32_T BusAddress; /* Bytes 8-11 */ - unsigned char Dummy2[4]; /* Bytes 12-15 */ - } __attribute__ ((packed)) Type3E; - struct { - DAC960_V1_CommandOpcode_T CommandOpcode; /* Byte 0 */ - DAC960_V1_CommandIdentifier_T CommandIdentifier; /* Byte 1 */ - unsigned char Dummy1[2]; /* Bytes 2-3 */ - unsigned char RebuildRateConstant; /* Byte 4 */ - unsigned char Dummy2[3]; /* Bytes 5-7 */ - DAC960_BusAddress32_T BusAddress; /* Bytes 8-11 */ - unsigned char Dummy3[4]; /* Bytes 12-15 */ - } __attribute__ ((packed)) Type3R; - struct { - DAC960_V1_CommandOpcode_T CommandOpcode; /* Byte 0 */ - DAC960_V1_CommandIdentifier_T CommandIdentifier; /* Byte 1 */ - unsigned short TransferLength; /* Bytes 2-3 */ - unsigned int LogicalBlockAddress; /* Bytes 4-7 */ - DAC960_BusAddress32_T BusAddress; /* Bytes 8-11 */ - unsigned char LogicalDriveNumber; /* Byte 12 */ - unsigned char Dummy[3]; /* Bytes 13-15 */ - } __attribute__ ((packed)) Type4; - struct { - DAC960_V1_CommandOpcode_T CommandOpcode; /* Byte 0 */ - DAC960_V1_CommandIdentifier_T CommandIdentifier; /* Byte 1 */ - struct { - unsigned short TransferLength:11; /* Bytes 2-3 */ - unsigned char LogicalDriveNumber:5; /* Byte 3 Bits 3-7 */ - } __attribute__ ((packed)) LD; - unsigned int LogicalBlockAddress; /* Bytes 4-7 */ - DAC960_BusAddress32_T BusAddress; /* Bytes 8-11 */ - unsigned char ScatterGatherCount:6; /* Byte 12 Bits 0-5 */ - enum { - DAC960_V1_ScatterGather_32BitAddress_32BitByteCount = 0x0, - DAC960_V1_ScatterGather_32BitAddress_16BitByteCount = 0x1, - DAC960_V1_ScatterGather_32BitByteCount_32BitAddress = 0x2, - DAC960_V1_ScatterGather_16BitByteCount_32BitAddress = 0x3 - } __attribute__ ((packed)) ScatterGatherType:2; /* Byte 12 Bits 6-7 */ - unsigned char Dummy[3]; /* Bytes 13-15 */ - } __attribute__ ((packed)) Type5; - struct { - DAC960_V1_CommandOpcode_T CommandOpcode; /* Byte 0 */ - DAC960_V1_CommandIdentifier_T CommandIdentifier; /* Byte 1 */ - unsigned char CommandOpcode2; /* Byte 2 */ - unsigned char :8; /* Byte 3 */ - DAC960_BusAddress32_T CommandMailboxesBusAddress; /* Bytes 4-7 */ - DAC960_BusAddress32_T StatusMailboxesBusAddress; /* Bytes 8-11 */ - unsigned char Dummy[4]; /* Bytes 12-15 */ - } __attribute__ ((packed)) TypeX; -} -DAC960_V1_CommandMailbox_T; - - -/* - Define the DAC960 V2 Firmware Command Opcodes. -*/ - -typedef enum -{ - DAC960_V2_MemCopy = 0x01, - DAC960_V2_SCSI_10_Passthru = 0x02, - DAC960_V2_SCSI_255_Passthru = 0x03, - DAC960_V2_SCSI_10 = 0x04, - DAC960_V2_SCSI_256 = 0x05, - DAC960_V2_IOCTL = 0x20 -} -__attribute__ ((packed)) -DAC960_V2_CommandOpcode_T; - - -/* - Define the DAC960 V2 Firmware IOCTL Opcodes. -*/ - -typedef enum -{ - DAC960_V2_GetControllerInfo = 0x01, - DAC960_V2_GetLogicalDeviceInfoValid = 0x03, - DAC960_V2_GetPhysicalDeviceInfoValid = 0x05, - DAC960_V2_GetHealthStatus = 0x11, - DAC960_V2_GetEvent = 0x15, - DAC960_V2_StartDiscovery = 0x81, - DAC960_V2_SetDeviceState = 0x82, - DAC960_V2_RebuildDeviceStart = 0x88, - DAC960_V2_RebuildDeviceStop = 0x89, - DAC960_V2_ConsistencyCheckStart = 0x8C, - DAC960_V2_ConsistencyCheckStop = 0x8D, - DAC960_V2_SetMemoryMailbox = 0x8E, - DAC960_V2_PauseDevice = 0x92, - DAC960_V2_TranslatePhysicalToLogicalDevice = 0xC5 -} -__attribute__ ((packed)) -DAC960_V2_IOCTL_Opcode_T; - - -/* - Define the DAC960 V2 Firmware Command Identifier type. -*/ - -typedef unsigned short DAC960_V2_CommandIdentifier_T; - - -/* - Define the DAC960 V2 Firmware Command Status Codes. -*/ - -#define DAC960_V2_NormalCompletion 0x00 -#define DAC960_V2_AbormalCompletion 0x02 -#define DAC960_V2_DeviceBusy 0x08 -#define DAC960_V2_DeviceNonresponsive 0x0E -#define DAC960_V2_DeviceNonresponsive2 0x0F -#define DAC960_V2_DeviceRevervationConflict 0x18 - -typedef unsigned char DAC960_V2_CommandStatus_T; - - -/* - Define the DAC960 V2 Firmware Memory Type structure. -*/ - -typedef struct DAC960_V2_MemoryType -{ - enum { - DAC960_V2_MemoryType_Reserved = 0x00, - DAC960_V2_MemoryType_DRAM = 0x01, - DAC960_V2_MemoryType_EDRAM = 0x02, - DAC960_V2_MemoryType_EDO = 0x03, - DAC960_V2_MemoryType_SDRAM = 0x04, - DAC960_V2_MemoryType_Last = 0x1F - } __attribute__ ((packed)) MemoryType:5; /* Byte 0 Bits 0-4 */ - bool :1; /* Byte 0 Bit 5 */ - bool MemoryParity:1; /* Byte 0 Bit 6 */ - bool MemoryECC:1; /* Byte 0 Bit 7 */ -} -DAC960_V2_MemoryType_T; - - -/* - Define the DAC960 V2 Firmware Processor Type structure. -*/ - -typedef enum -{ - DAC960_V2_ProcessorType_i960CA = 0x01, - DAC960_V2_ProcessorType_i960RD = 0x02, - DAC960_V2_ProcessorType_i960RN = 0x03, - DAC960_V2_ProcessorType_i960RP = 0x04, - DAC960_V2_ProcessorType_NorthBay = 0x05, - DAC960_V2_ProcessorType_StrongArm = 0x06, - DAC960_V2_ProcessorType_i960RM = 0x07 -} -__attribute__ ((packed)) -DAC960_V2_ProcessorType_T; - - -/* - Define the DAC960 V2 Firmware Get Controller Info reply structure. -*/ - -typedef struct DAC960_V2_ControllerInfo -{ - unsigned char :8; /* Byte 0 */ - enum { - DAC960_V2_SCSI_Bus = 0x00, - DAC960_V2_Fibre_Bus = 0x01, - DAC960_V2_PCI_Bus = 0x03 - } __attribute__ ((packed)) BusInterfaceType; /* Byte 1 */ - enum { - DAC960_V2_DAC960E = 0x01, - DAC960_V2_DAC960M = 0x08, - DAC960_V2_DAC960PD = 0x10, - DAC960_V2_DAC960PL = 0x11, - DAC960_V2_DAC960PU = 0x12, - DAC960_V2_DAC960PE = 0x13, - DAC960_V2_DAC960PG = 0x14, - DAC960_V2_DAC960PJ = 0x15, - DAC960_V2_DAC960PTL0 = 0x16, - DAC960_V2_DAC960PR = 0x17, - DAC960_V2_DAC960PRL = 0x18, - DAC960_V2_DAC960PT = 0x19, - DAC960_V2_DAC1164P = 0x1A, - DAC960_V2_DAC960PTL1 = 0x1B, - DAC960_V2_EXR2000P = 0x1C, - DAC960_V2_EXR3000P = 0x1D, - DAC960_V2_AcceleRAID352 = 0x1E, - DAC960_V2_AcceleRAID170 = 0x1F, - DAC960_V2_AcceleRAID160 = 0x20, - DAC960_V2_DAC960S = 0x60, - DAC960_V2_DAC960SU = 0x61, - DAC960_V2_DAC960SX = 0x62, - DAC960_V2_DAC960SF = 0x63, - DAC960_V2_DAC960SS = 0x64, - DAC960_V2_DAC960FL = 0x65, - DAC960_V2_DAC960LL = 0x66, - DAC960_V2_DAC960FF = 0x67, - DAC960_V2_DAC960HP = 0x68, - DAC960_V2_RAIDBRICK = 0x69, - DAC960_V2_METEOR_FL = 0x6A, - DAC960_V2_METEOR_FF = 0x6B - } __attribute__ ((packed)) ControllerType; /* Byte 2 */ - unsigned char :8; /* Byte 3 */ - unsigned short BusInterfaceSpeedMHz; /* Bytes 4-5 */ - unsigned char BusWidthBits; /* Byte 6 */ - unsigned char FlashCodeTypeOrProductID; /* Byte 7 */ - unsigned char NumberOfHostPortsPresent; /* Byte 8 */ - unsigned char Reserved1[7]; /* Bytes 9-15 */ - unsigned char BusInterfaceName[16]; /* Bytes 16-31 */ - unsigned char ControllerName[16]; /* Bytes 32-47 */ - unsigned char Reserved2[16]; /* Bytes 48-63 */ - /* Firmware Release Information */ - unsigned char FirmwareMajorVersion; /* Byte 64 */ - unsigned char FirmwareMinorVersion; /* Byte 65 */ - unsigned char FirmwareTurnNumber; /* Byte 66 */ - unsigned char FirmwareBuildNumber; /* Byte 67 */ - unsigned char FirmwareReleaseDay; /* Byte 68 */ - unsigned char FirmwareReleaseMonth; /* Byte 69 */ - unsigned char FirmwareReleaseYearHigh2Digits; /* Byte 70 */ - unsigned char FirmwareReleaseYearLow2Digits; /* Byte 71 */ - /* Hardware Release Information */ - unsigned char HardwareRevision; /* Byte 72 */ - unsigned int :24; /* Bytes 73-75 */ - unsigned char HardwareReleaseDay; /* Byte 76 */ - unsigned char HardwareReleaseMonth; /* Byte 77 */ - unsigned char HardwareReleaseYearHigh2Digits; /* Byte 78 */ - unsigned char HardwareReleaseYearLow2Digits; /* Byte 79 */ - /* Hardware Manufacturing Information */ - unsigned char ManufacturingBatchNumber; /* Byte 80 */ - unsigned char :8; /* Byte 81 */ - unsigned char ManufacturingPlantNumber; /* Byte 82 */ - unsigned char :8; /* Byte 83 */ - unsigned char HardwareManufacturingDay; /* Byte 84 */ - unsigned char HardwareManufacturingMonth; /* Byte 85 */ - unsigned char HardwareManufacturingYearHigh2Digits; /* Byte 86 */ - unsigned char HardwareManufacturingYearLow2Digits; /* Byte 87 */ - unsigned char MaximumNumberOfPDDperXLD; /* Byte 88 */ - unsigned char MaximumNumberOfILDperXLD; /* Byte 89 */ - unsigned short NonvolatileMemorySizeKB; /* Bytes 90-91 */ - unsigned char MaximumNumberOfXLD; /* Byte 92 */ - unsigned int :24; /* Bytes 93-95 */ - /* Unique Information per Controller */ - unsigned char ControllerSerialNumber[16]; /* Bytes 96-111 */ - unsigned char Reserved3[16]; /* Bytes 112-127 */ - /* Vendor Information */ - unsigned int :24; /* Bytes 128-130 */ - unsigned char OEM_Code; /* Byte 131 */ - unsigned char VendorName[16]; /* Bytes 132-147 */ - /* Other Physical/Controller/Operation Information */ - bool BBU_Present:1; /* Byte 148 Bit 0 */ - bool ActiveActiveClusteringMode:1; /* Byte 148 Bit 1 */ - unsigned char :6; /* Byte 148 Bits 2-7 */ - unsigned char :8; /* Byte 149 */ - unsigned short :16; /* Bytes 150-151 */ - /* Physical Device Scan Information */ - bool PhysicalScanActive:1; /* Byte 152 Bit 0 */ - unsigned char :7; /* Byte 152 Bits 1-7 */ - unsigned char PhysicalDeviceChannelNumber; /* Byte 153 */ - unsigned char PhysicalDeviceTargetID; /* Byte 154 */ - unsigned char PhysicalDeviceLogicalUnit; /* Byte 155 */ - /* Maximum Command Data Transfer Sizes */ - unsigned short MaximumDataTransferSizeInBlocks; /* Bytes 156-157 */ - unsigned short MaximumScatterGatherEntries; /* Bytes 158-159 */ - /* Logical/Physical Device Counts */ - unsigned short LogicalDevicesPresent; /* Bytes 160-161 */ - unsigned short LogicalDevicesCritical; /* Bytes 162-163 */ - unsigned short LogicalDevicesOffline; /* Bytes 164-165 */ - unsigned short PhysicalDevicesPresent; /* Bytes 166-167 */ - unsigned short PhysicalDisksPresent; /* Bytes 168-169 */ - unsigned short PhysicalDisksCritical; /* Bytes 170-171 */ - unsigned short PhysicalDisksOffline; /* Bytes 172-173 */ - unsigned short MaximumParallelCommands; /* Bytes 174-175 */ - /* Channel and Target ID Information */ - unsigned char NumberOfPhysicalChannelsPresent; /* Byte 176 */ - unsigned char NumberOfVirtualChannelsPresent; /* Byte 177 */ - unsigned char NumberOfPhysicalChannelsPossible; /* Byte 178 */ - unsigned char NumberOfVirtualChannelsPossible; /* Byte 179 */ - unsigned char MaximumTargetsPerChannel[16]; /* Bytes 180-195 */ - unsigned char Reserved4[12]; /* Bytes 196-207 */ - /* Memory/Cache Information */ - unsigned short MemorySizeMB; /* Bytes 208-209 */ - unsigned short CacheSizeMB; /* Bytes 210-211 */ - unsigned int ValidCacheSizeInBytes; /* Bytes 212-215 */ - unsigned int DirtyCacheSizeInBytes; /* Bytes 216-219 */ - unsigned short MemorySpeedMHz; /* Bytes 220-221 */ - unsigned char MemoryDataWidthBits; /* Byte 222 */ - DAC960_V2_MemoryType_T MemoryType; /* Byte 223 */ - unsigned char CacheMemoryTypeName[16]; /* Bytes 224-239 */ - /* Execution Memory Information */ - unsigned short ExecutionMemorySizeMB; /* Bytes 240-241 */ - unsigned short ExecutionL2CacheSizeMB; /* Bytes 242-243 */ - unsigned char Reserved5[8]; /* Bytes 244-251 */ - unsigned short ExecutionMemorySpeedMHz; /* Bytes 252-253 */ - unsigned char ExecutionMemoryDataWidthBits; /* Byte 254 */ - DAC960_V2_MemoryType_T ExecutionMemoryType; /* Byte 255 */ - unsigned char ExecutionMemoryTypeName[16]; /* Bytes 256-271 */ - /* First CPU Type Information */ - unsigned short FirstProcessorSpeedMHz; /* Bytes 272-273 */ - DAC960_V2_ProcessorType_T FirstProcessorType; /* Byte 274 */ - unsigned char FirstProcessorCount; /* Byte 275 */ - unsigned char Reserved6[12]; /* Bytes 276-287 */ - unsigned char FirstProcessorName[16]; /* Bytes 288-303 */ - /* Second CPU Type Information */ - unsigned short SecondProcessorSpeedMHz; /* Bytes 304-305 */ - DAC960_V2_ProcessorType_T SecondProcessorType; /* Byte 306 */ - unsigned char SecondProcessorCount; /* Byte 307 */ - unsigned char Reserved7[12]; /* Bytes 308-319 */ - unsigned char SecondProcessorName[16]; /* Bytes 320-335 */ - /* Debugging/Profiling/Command Time Tracing Information */ - unsigned short CurrentProfilingDataPageNumber; /* Bytes 336-337 */ - unsigned short ProgramsAwaitingProfilingData; /* Bytes 338-339 */ - unsigned short CurrentCommandTimeTraceDataPageNumber; /* Bytes 340-341 */ - unsigned short ProgramsAwaitingCommandTimeTraceData; /* Bytes 342-343 */ - unsigned char Reserved8[8]; /* Bytes 344-351 */ - /* Error Counters on Physical Devices */ - unsigned short PhysicalDeviceBusResets; /* Bytes 352-353 */ - unsigned short PhysicalDeviceParityErrors; /* Bytes 355-355 */ - unsigned short PhysicalDeviceSoftErrors; /* Bytes 356-357 */ - unsigned short PhysicalDeviceCommandsFailed; /* Bytes 358-359 */ - unsigned short PhysicalDeviceMiscellaneousErrors; /* Bytes 360-361 */ - unsigned short PhysicalDeviceCommandTimeouts; /* Bytes 362-363 */ - unsigned short PhysicalDeviceSelectionTimeouts; /* Bytes 364-365 */ - unsigned short PhysicalDeviceRetriesDone; /* Bytes 366-367 */ - unsigned short PhysicalDeviceAbortsDone; /* Bytes 368-369 */ - unsigned short PhysicalDeviceHostCommandAbortsDone; /* Bytes 370-371 */ - unsigned short PhysicalDevicePredictedFailuresDetected; /* Bytes 372-373 */ - unsigned short PhysicalDeviceHostCommandsFailed; /* Bytes 374-375 */ - unsigned short PhysicalDeviceHardErrors; /* Bytes 376-377 */ - unsigned char Reserved9[6]; /* Bytes 378-383 */ - /* Error Counters on Logical Devices */ - unsigned short LogicalDeviceSoftErrors; /* Bytes 384-385 */ - unsigned short LogicalDeviceCommandsFailed; /* Bytes 386-387 */ - unsigned short LogicalDeviceHostCommandAbortsDone; /* Bytes 388-389 */ - unsigned short :16; /* Bytes 390-391 */ - /* Error Counters on Controller */ - unsigned short ControllerMemoryErrors; /* Bytes 392-393 */ - unsigned short ControllerHostCommandAbortsDone; /* Bytes 394-395 */ - unsigned int :32; /* Bytes 396-399 */ - /* Long Duration Activity Information */ - unsigned short BackgroundInitializationsActive; /* Bytes 400-401 */ - unsigned short LogicalDeviceInitializationsActive; /* Bytes 402-403 */ - unsigned short PhysicalDeviceInitializationsActive; /* Bytes 404-405 */ - unsigned short ConsistencyChecksActive; /* Bytes 406-407 */ - unsigned short RebuildsActive; /* Bytes 408-409 */ - unsigned short OnlineExpansionsActive; /* Bytes 410-411 */ - unsigned short PatrolActivitiesActive; /* Bytes 412-413 */ - unsigned short :16; /* Bytes 414-415 */ - /* Flash ROM Information */ - unsigned char FlashType; /* Byte 416 */ - unsigned char :8; /* Byte 417 */ - unsigned short FlashSizeMB; /* Bytes 418-419 */ - unsigned int FlashLimit; /* Bytes 420-423 */ - unsigned int FlashCount; /* Bytes 424-427 */ - unsigned int :32; /* Bytes 428-431 */ - unsigned char FlashTypeName[16]; /* Bytes 432-447 */ - /* Firmware Run Time Information */ - unsigned char RebuildRate; /* Byte 448 */ - unsigned char BackgroundInitializationRate; /* Byte 449 */ - unsigned char ForegroundInitializationRate; /* Byte 450 */ - unsigned char ConsistencyCheckRate; /* Byte 451 */ - unsigned int :32; /* Bytes 452-455 */ - unsigned int MaximumDP; /* Bytes 456-459 */ - unsigned int FreeDP; /* Bytes 460-463 */ - unsigned int MaximumIOP; /* Bytes 464-467 */ - unsigned int FreeIOP; /* Bytes 468-471 */ - unsigned short MaximumCombLengthInBlocks; /* Bytes 472-473 */ - unsigned short NumberOfConfigurationGroups; /* Bytes 474-475 */ - bool InstallationAbortStatus:1; /* Byte 476 Bit 0 */ - bool MaintenanceModeStatus:1; /* Byte 476 Bit 1 */ - unsigned int :24; /* Bytes 476-479 */ - unsigned char Reserved10[32]; /* Bytes 480-511 */ - unsigned char Reserved11[512]; /* Bytes 512-1023 */ -} -DAC960_V2_ControllerInfo_T; - - -/* - Define the DAC960 V2 Firmware Logical Device State type. -*/ - -typedef enum -{ - DAC960_V2_LogicalDevice_Online = 0x01, - DAC960_V2_LogicalDevice_Offline = 0x08, - DAC960_V2_LogicalDevice_Critical = 0x09 -} -__attribute__ ((packed)) -DAC960_V2_LogicalDeviceState_T; - - -/* - Define the DAC960 V2 Firmware Get Logical Device Info reply structure. -*/ - -typedef struct DAC960_V2_LogicalDeviceInfo -{ - unsigned char :8; /* Byte 0 */ - unsigned char Channel; /* Byte 1 */ - unsigned char TargetID; /* Byte 2 */ - unsigned char LogicalUnit; /* Byte 3 */ - DAC960_V2_LogicalDeviceState_T LogicalDeviceState; /* Byte 4 */ - unsigned char RAIDLevel; /* Byte 5 */ - unsigned char StripeSize; /* Byte 6 */ - unsigned char CacheLineSize; /* Byte 7 */ - struct { - enum { - DAC960_V2_ReadCacheDisabled = 0x0, - DAC960_V2_ReadCacheEnabled = 0x1, - DAC960_V2_ReadAheadEnabled = 0x2, - DAC960_V2_IntelligentReadAheadEnabled = 0x3, - DAC960_V2_ReadCache_Last = 0x7 - } __attribute__ ((packed)) ReadCache:3; /* Byte 8 Bits 0-2 */ - enum { - DAC960_V2_WriteCacheDisabled = 0x0, - DAC960_V2_LogicalDeviceReadOnly = 0x1, - DAC960_V2_WriteCacheEnabled = 0x2, - DAC960_V2_IntelligentWriteCacheEnabled = 0x3, - DAC960_V2_WriteCache_Last = 0x7 - } __attribute__ ((packed)) WriteCache:3; /* Byte 8 Bits 3-5 */ - bool :1; /* Byte 8 Bit 6 */ - bool LogicalDeviceInitialized:1; /* Byte 8 Bit 7 */ - } LogicalDeviceControl; /* Byte 8 */ - /* Logical Device Operations Status */ - bool ConsistencyCheckInProgress:1; /* Byte 9 Bit 0 */ - bool RebuildInProgress:1; /* Byte 9 Bit 1 */ - bool BackgroundInitializationInProgress:1; /* Byte 9 Bit 2 */ - bool ForegroundInitializationInProgress:1; /* Byte 9 Bit 3 */ - bool DataMigrationInProgress:1; /* Byte 9 Bit 4 */ - bool PatrolOperationInProgress:1; /* Byte 9 Bit 5 */ - unsigned char :2; /* Byte 9 Bits 6-7 */ - unsigned char RAID5WriteUpdate; /* Byte 10 */ - unsigned char RAID5Algorithm; /* Byte 11 */ - unsigned short LogicalDeviceNumber; /* Bytes 12-13 */ - /* BIOS Info */ - bool BIOSDisabled:1; /* Byte 14 Bit 0 */ - bool CDROMBootEnabled:1; /* Byte 14 Bit 1 */ - bool DriveCoercionEnabled:1; /* Byte 14 Bit 2 */ - bool WriteSameDisabled:1; /* Byte 14 Bit 3 */ - bool HBA_ModeEnabled:1; /* Byte 14 Bit 4 */ - enum { - DAC960_V2_Geometry_128_32 = 0x0, - DAC960_V2_Geometry_255_63 = 0x1, - DAC960_V2_Geometry_Reserved1 = 0x2, - DAC960_V2_Geometry_Reserved2 = 0x3 - } __attribute__ ((packed)) DriveGeometry:2; /* Byte 14 Bits 5-6 */ - bool SuperReadAheadEnabled:1; /* Byte 14 Bit 7 */ - unsigned char :8; /* Byte 15 */ - /* Error Counters */ - unsigned short SoftErrors; /* Bytes 16-17 */ - unsigned short CommandsFailed; /* Bytes 18-19 */ - unsigned short HostCommandAbortsDone; /* Bytes 20-21 */ - unsigned short DeferredWriteErrors; /* Bytes 22-23 */ - unsigned int :32; /* Bytes 24-27 */ - unsigned int :32; /* Bytes 28-31 */ - /* Device Size Information */ - unsigned short :16; /* Bytes 32-33 */ - unsigned short DeviceBlockSizeInBytes; /* Bytes 34-35 */ - unsigned int OriginalDeviceSize; /* Bytes 36-39 */ - unsigned int ConfigurableDeviceSize; /* Bytes 40-43 */ - unsigned int :32; /* Bytes 44-47 */ - unsigned char LogicalDeviceName[32]; /* Bytes 48-79 */ - unsigned char SCSI_InquiryData[36]; /* Bytes 80-115 */ - unsigned char Reserved1[12]; /* Bytes 116-127 */ - DAC960_ByteCount64_T LastReadBlockNumber; /* Bytes 128-135 */ - DAC960_ByteCount64_T LastWrittenBlockNumber; /* Bytes 136-143 */ - DAC960_ByteCount64_T ConsistencyCheckBlockNumber; /* Bytes 144-151 */ - DAC960_ByteCount64_T RebuildBlockNumber; /* Bytes 152-159 */ - DAC960_ByteCount64_T BackgroundInitializationBlockNumber; /* Bytes 160-167 */ - DAC960_ByteCount64_T ForegroundInitializationBlockNumber; /* Bytes 168-175 */ - DAC960_ByteCount64_T DataMigrationBlockNumber; /* Bytes 176-183 */ - DAC960_ByteCount64_T PatrolOperationBlockNumber; /* Bytes 184-191 */ - unsigned char Reserved2[64]; /* Bytes 192-255 */ -} -DAC960_V2_LogicalDeviceInfo_T; - - -/* - Define the DAC960 V2 Firmware Physical Device State type. -*/ - -typedef enum -{ - DAC960_V2_Device_Unconfigured = 0x00, - DAC960_V2_Device_Online = 0x01, - DAC960_V2_Device_Rebuild = 0x03, - DAC960_V2_Device_Missing = 0x04, - DAC960_V2_Device_Critical = 0x05, - DAC960_V2_Device_Dead = 0x08, - DAC960_V2_Device_SuspectedDead = 0x0C, - DAC960_V2_Device_CommandedOffline = 0x10, - DAC960_V2_Device_Standby = 0x21, - DAC960_V2_Device_InvalidState = 0xFF -} -__attribute__ ((packed)) -DAC960_V2_PhysicalDeviceState_T; - - -/* - Define the DAC960 V2 Firmware Get Physical Device Info reply structure. -*/ - -typedef struct DAC960_V2_PhysicalDeviceInfo -{ - unsigned char :8; /* Byte 0 */ - unsigned char Channel; /* Byte 1 */ - unsigned char TargetID; /* Byte 2 */ - unsigned char LogicalUnit; /* Byte 3 */ - /* Configuration Status Bits */ - bool PhysicalDeviceFaultTolerant:1; /* Byte 4 Bit 0 */ - bool PhysicalDeviceConnected:1; /* Byte 4 Bit 1 */ - bool PhysicalDeviceLocalToController:1; /* Byte 4 Bit 2 */ - unsigned char :5; /* Byte 4 Bits 3-7 */ - /* Multiple Host/Controller Status Bits */ - bool RemoteHostSystemDead:1; /* Byte 5 Bit 0 */ - bool RemoteControllerDead:1; /* Byte 5 Bit 1 */ - unsigned char :6; /* Byte 5 Bits 2-7 */ - DAC960_V2_PhysicalDeviceState_T PhysicalDeviceState; /* Byte 6 */ - unsigned char NegotiatedDataWidthBits; /* Byte 7 */ - unsigned short NegotiatedSynchronousMegaTransfers; /* Bytes 8-9 */ - /* Multiported Physical Device Information */ - unsigned char NumberOfPortConnections; /* Byte 10 */ - unsigned char DriveAccessibilityBitmap; /* Byte 11 */ - unsigned int :32; /* Bytes 12-15 */ - unsigned char NetworkAddress[16]; /* Bytes 16-31 */ - unsigned short MaximumTags; /* Bytes 32-33 */ - /* Physical Device Operations Status */ - bool ConsistencyCheckInProgress:1; /* Byte 34 Bit 0 */ - bool RebuildInProgress:1; /* Byte 34 Bit 1 */ - bool MakingDataConsistentInProgress:1; /* Byte 34 Bit 2 */ - bool PhysicalDeviceInitializationInProgress:1; /* Byte 34 Bit 3 */ - bool DataMigrationInProgress:1; /* Byte 34 Bit 4 */ - bool PatrolOperationInProgress:1; /* Byte 34 Bit 5 */ - unsigned char :2; /* Byte 34 Bits 6-7 */ - unsigned char LongOperationStatus; /* Byte 35 */ - unsigned char ParityErrors; /* Byte 36 */ - unsigned char SoftErrors; /* Byte 37 */ - unsigned char HardErrors; /* Byte 38 */ - unsigned char MiscellaneousErrors; /* Byte 39 */ - unsigned char CommandTimeouts; /* Byte 40 */ - unsigned char Retries; /* Byte 41 */ - unsigned char Aborts; /* Byte 42 */ - unsigned char PredictedFailuresDetected; /* Byte 43 */ - unsigned int :32; /* Bytes 44-47 */ - unsigned short :16; /* Bytes 48-49 */ - unsigned short DeviceBlockSizeInBytes; /* Bytes 50-51 */ - unsigned int OriginalDeviceSize; /* Bytes 52-55 */ - unsigned int ConfigurableDeviceSize; /* Bytes 56-59 */ - unsigned int :32; /* Bytes 60-63 */ - unsigned char PhysicalDeviceName[16]; /* Bytes 64-79 */ - unsigned char Reserved1[16]; /* Bytes 80-95 */ - unsigned char Reserved2[32]; /* Bytes 96-127 */ - unsigned char SCSI_InquiryData[36]; /* Bytes 128-163 */ - unsigned char Reserved3[20]; /* Bytes 164-183 */ - unsigned char Reserved4[8]; /* Bytes 184-191 */ - DAC960_ByteCount64_T LastReadBlockNumber; /* Bytes 192-199 */ - DAC960_ByteCount64_T LastWrittenBlockNumber; /* Bytes 200-207 */ - DAC960_ByteCount64_T ConsistencyCheckBlockNumber; /* Bytes 208-215 */ - DAC960_ByteCount64_T RebuildBlockNumber; /* Bytes 216-223 */ - DAC960_ByteCount64_T MakingDataConsistentBlockNumber; /* Bytes 224-231 */ - DAC960_ByteCount64_T DeviceInitializationBlockNumber; /* Bytes 232-239 */ - DAC960_ByteCount64_T DataMigrationBlockNumber; /* Bytes 240-247 */ - DAC960_ByteCount64_T PatrolOperationBlockNumber; /* Bytes 248-255 */ - unsigned char Reserved5[256]; /* Bytes 256-511 */ -} -DAC960_V2_PhysicalDeviceInfo_T; - - -/* - Define the DAC960 V2 Firmware Health Status Buffer structure. -*/ - -typedef struct DAC960_V2_HealthStatusBuffer -{ - unsigned int MicrosecondsFromControllerStartTime; /* Bytes 0-3 */ - unsigned int MillisecondsFromControllerStartTime; /* Bytes 4-7 */ - unsigned int SecondsFrom1January1970; /* Bytes 8-11 */ - unsigned int :32; /* Bytes 12-15 */ - unsigned int StatusChangeCounter; /* Bytes 16-19 */ - unsigned int :32; /* Bytes 20-23 */ - unsigned int DebugOutputMessageBufferIndex; /* Bytes 24-27 */ - unsigned int CodedMessageBufferIndex; /* Bytes 28-31 */ - unsigned int CurrentTimeTracePageNumber; /* Bytes 32-35 */ - unsigned int CurrentProfilerPageNumber; /* Bytes 36-39 */ - unsigned int NextEventSequenceNumber; /* Bytes 40-43 */ - unsigned int :32; /* Bytes 44-47 */ - unsigned char Reserved1[16]; /* Bytes 48-63 */ - unsigned char Reserved2[64]; /* Bytes 64-127 */ -} -DAC960_V2_HealthStatusBuffer_T; - - -/* - Define the DAC960 V2 Firmware Get Event reply structure. -*/ - -typedef struct DAC960_V2_Event -{ - unsigned int EventSequenceNumber; /* Bytes 0-3 */ - unsigned int EventTime; /* Bytes 4-7 */ - unsigned int EventCode; /* Bytes 8-11 */ - unsigned char :8; /* Byte 12 */ - unsigned char Channel; /* Byte 13 */ - unsigned char TargetID; /* Byte 14 */ - unsigned char LogicalUnit; /* Byte 15 */ - unsigned int :32; /* Bytes 16-19 */ - unsigned int EventSpecificParameter; /* Bytes 20-23 */ - unsigned char RequestSenseData[40]; /* Bytes 24-63 */ -} -DAC960_V2_Event_T; - - -/* - Define the DAC960 V2 Firmware Command Control Bits structure. -*/ - -typedef struct DAC960_V2_CommandControlBits -{ - bool ForceUnitAccess:1; /* Byte 0 Bit 0 */ - bool DisablePageOut:1; /* Byte 0 Bit 1 */ - bool :1; /* Byte 0 Bit 2 */ - bool AdditionalScatterGatherListMemory:1; /* Byte 0 Bit 3 */ - bool DataTransferControllerToHost:1; /* Byte 0 Bit 4 */ - bool :1; /* Byte 0 Bit 5 */ - bool NoAutoRequestSense:1; /* Byte 0 Bit 6 */ - bool DisconnectProhibited:1; /* Byte 0 Bit 7 */ -} -DAC960_V2_CommandControlBits_T; - - -/* - Define the DAC960 V2 Firmware Command Timeout structure. -*/ - -typedef struct DAC960_V2_CommandTimeout -{ - unsigned char TimeoutValue:6; /* Byte 0 Bits 0-5 */ - enum { - DAC960_V2_TimeoutScale_Seconds = 0, - DAC960_V2_TimeoutScale_Minutes = 1, - DAC960_V2_TimeoutScale_Hours = 2, - DAC960_V2_TimeoutScale_Reserved = 3 - } __attribute__ ((packed)) TimeoutScale:2; /* Byte 0 Bits 6-7 */ -} -DAC960_V2_CommandTimeout_T; - - -/* - Define the DAC960 V2 Firmware Physical Device structure. -*/ - -typedef struct DAC960_V2_PhysicalDevice -{ - unsigned char LogicalUnit; /* Byte 0 */ - unsigned char TargetID; /* Byte 1 */ - unsigned char Channel:3; /* Byte 2 Bits 0-2 */ - unsigned char Controller:5; /* Byte 2 Bits 3-7 */ -} -__attribute__ ((packed)) -DAC960_V2_PhysicalDevice_T; - - -/* - Define the DAC960 V2 Firmware Logical Device structure. -*/ - -typedef struct DAC960_V2_LogicalDevice -{ - unsigned short LogicalDeviceNumber; /* Bytes 0-1 */ - unsigned char :3; /* Byte 2 Bits 0-2 */ - unsigned char Controller:5; /* Byte 2 Bits 3-7 */ -} -__attribute__ ((packed)) -DAC960_V2_LogicalDevice_T; - - -/* - Define the DAC960 V2 Firmware Operation Device type. -*/ - -typedef enum -{ - DAC960_V2_Physical_Device = 0x00, - DAC960_V2_RAID_Device = 0x01, - DAC960_V2_Physical_Channel = 0x02, - DAC960_V2_RAID_Channel = 0x03, - DAC960_V2_Physical_Controller = 0x04, - DAC960_V2_RAID_Controller = 0x05, - DAC960_V2_Configuration_Group = 0x10, - DAC960_V2_Enclosure = 0x11 -} -__attribute__ ((packed)) -DAC960_V2_OperationDevice_T; - - -/* - Define the DAC960 V2 Firmware Translate Physical To Logical Device structure. -*/ - -typedef struct DAC960_V2_PhysicalToLogicalDevice -{ - unsigned short LogicalDeviceNumber; /* Bytes 0-1 */ - unsigned short :16; /* Bytes 2-3 */ - unsigned char PreviousBootController; /* Byte 4 */ - unsigned char PreviousBootChannel; /* Byte 5 */ - unsigned char PreviousBootTargetID; /* Byte 6 */ - unsigned char PreviousBootLogicalUnit; /* Byte 7 */ -} -DAC960_V2_PhysicalToLogicalDevice_T; - - - -/* - Define the DAC960 V2 Firmware Scatter/Gather List Entry structure. -*/ - -typedef struct DAC960_V2_ScatterGatherSegment -{ - DAC960_BusAddress64_T SegmentDataPointer; /* Bytes 0-7 */ - DAC960_ByteCount64_T SegmentByteCount; /* Bytes 8-15 */ -} -DAC960_V2_ScatterGatherSegment_T; - - -/* - Define the DAC960 V2 Firmware Data Transfer Memory Address structure. -*/ - -typedef union DAC960_V2_DataTransferMemoryAddress -{ - DAC960_V2_ScatterGatherSegment_T ScatterGatherSegments[2]; /* Bytes 0-31 */ - struct { - unsigned short ScatterGatherList0Length; /* Bytes 0-1 */ - unsigned short ScatterGatherList1Length; /* Bytes 2-3 */ - unsigned short ScatterGatherList2Length; /* Bytes 4-5 */ - unsigned short :16; /* Bytes 6-7 */ - DAC960_BusAddress64_T ScatterGatherList0Address; /* Bytes 8-15 */ - DAC960_BusAddress64_T ScatterGatherList1Address; /* Bytes 16-23 */ - DAC960_BusAddress64_T ScatterGatherList2Address; /* Bytes 24-31 */ - } ExtendedScatterGather; -} -DAC960_V2_DataTransferMemoryAddress_T; - - -/* - Define the 64 Byte DAC960 V2 Firmware Command Mailbox structure. -*/ - -typedef union DAC960_V2_CommandMailbox -{ - unsigned int Words[16]; /* Words 0-15 */ - struct { - DAC960_V2_CommandIdentifier_T CommandIdentifier; /* Bytes 0-1 */ - DAC960_V2_CommandOpcode_T CommandOpcode; /* Byte 2 */ - DAC960_V2_CommandControlBits_T CommandControlBits; /* Byte 3 */ - DAC960_ByteCount32_T DataTransferSize:24; /* Bytes 4-6 */ - unsigned char DataTransferPageNumber; /* Byte 7 */ - DAC960_BusAddress64_T RequestSenseBusAddress; /* Bytes 8-15 */ - unsigned int :24; /* Bytes 16-18 */ - DAC960_V2_CommandTimeout_T CommandTimeout; /* Byte 19 */ - unsigned char RequestSenseSize; /* Byte 20 */ - unsigned char IOCTL_Opcode; /* Byte 21 */ - unsigned char Reserved[10]; /* Bytes 22-31 */ - DAC960_V2_DataTransferMemoryAddress_T - DataTransferMemoryAddress; /* Bytes 32-63 */ - } Common; - struct { - DAC960_V2_CommandIdentifier_T CommandIdentifier; /* Bytes 0-1 */ - DAC960_V2_CommandOpcode_T CommandOpcode; /* Byte 2 */ - DAC960_V2_CommandControlBits_T CommandControlBits; /* Byte 3 */ - DAC960_ByteCount32_T DataTransferSize; /* Bytes 4-7 */ - DAC960_BusAddress64_T RequestSenseBusAddress; /* Bytes 8-15 */ - DAC960_V2_PhysicalDevice_T PhysicalDevice; /* Bytes 16-18 */ - DAC960_V2_CommandTimeout_T CommandTimeout; /* Byte 19 */ - unsigned char RequestSenseSize; /* Byte 20 */ - unsigned char CDBLength; /* Byte 21 */ - unsigned char SCSI_CDB[10]; /* Bytes 22-31 */ - DAC960_V2_DataTransferMemoryAddress_T - DataTransferMemoryAddress; /* Bytes 32-63 */ - } SCSI_10; - struct { - DAC960_V2_CommandIdentifier_T CommandIdentifier; /* Bytes 0-1 */ - DAC960_V2_CommandOpcode_T CommandOpcode; /* Byte 2 */ - DAC960_V2_CommandControlBits_T CommandControlBits; /* Byte 3 */ - DAC960_ByteCount32_T DataTransferSize; /* Bytes 4-7 */ - DAC960_BusAddress64_T RequestSenseBusAddress; /* Bytes 8-15 */ - DAC960_V2_PhysicalDevice_T PhysicalDevice; /* Bytes 16-18 */ - DAC960_V2_CommandTimeout_T CommandTimeout; /* Byte 19 */ - unsigned char RequestSenseSize; /* Byte 20 */ - unsigned char CDBLength; /* Byte 21 */ - unsigned short :16; /* Bytes 22-23 */ - DAC960_BusAddress64_T SCSI_CDB_BusAddress; /* Bytes 24-31 */ - DAC960_V2_DataTransferMemoryAddress_T - DataTransferMemoryAddress; /* Bytes 32-63 */ - } SCSI_255; - struct { - DAC960_V2_CommandIdentifier_T CommandIdentifier; /* Bytes 0-1 */ - DAC960_V2_CommandOpcode_T CommandOpcode; /* Byte 2 */ - DAC960_V2_CommandControlBits_T CommandControlBits; /* Byte 3 */ - DAC960_ByteCount32_T DataTransferSize:24; /* Bytes 4-6 */ - unsigned char DataTransferPageNumber; /* Byte 7 */ - DAC960_BusAddress64_T RequestSenseBusAddress; /* Bytes 8-15 */ - unsigned short :16; /* Bytes 16-17 */ - unsigned char ControllerNumber; /* Byte 18 */ - DAC960_V2_CommandTimeout_T CommandTimeout; /* Byte 19 */ - unsigned char RequestSenseSize; /* Byte 20 */ - unsigned char IOCTL_Opcode; /* Byte 21 */ - unsigned char Reserved[10]; /* Bytes 22-31 */ - DAC960_V2_DataTransferMemoryAddress_T - DataTransferMemoryAddress; /* Bytes 32-63 */ - } ControllerInfo; - struct { - DAC960_V2_CommandIdentifier_T CommandIdentifier; /* Bytes 0-1 */ - DAC960_V2_CommandOpcode_T CommandOpcode; /* Byte 2 */ - DAC960_V2_CommandControlBits_T CommandControlBits; /* Byte 3 */ - DAC960_ByteCount32_T DataTransferSize:24; /* Bytes 4-6 */ - unsigned char DataTransferPageNumber; /* Byte 7 */ - DAC960_BusAddress64_T RequestSenseBusAddress; /* Bytes 8-15 */ - DAC960_V2_LogicalDevice_T LogicalDevice; /* Bytes 16-18 */ - DAC960_V2_CommandTimeout_T CommandTimeout; /* Byte 19 */ - unsigned char RequestSenseSize; /* Byte 20 */ - unsigned char IOCTL_Opcode; /* Byte 21 */ - unsigned char Reserved[10]; /* Bytes 22-31 */ - DAC960_V2_DataTransferMemoryAddress_T - DataTransferMemoryAddress; /* Bytes 32-63 */ - } LogicalDeviceInfo; - struct { - DAC960_V2_CommandIdentifier_T CommandIdentifier; /* Bytes 0-1 */ - DAC960_V2_CommandOpcode_T CommandOpcode; /* Byte 2 */ - DAC960_V2_CommandControlBits_T CommandControlBits; /* Byte 3 */ - DAC960_ByteCount32_T DataTransferSize:24; /* Bytes 4-6 */ - unsigned char DataTransferPageNumber; /* Byte 7 */ - DAC960_BusAddress64_T RequestSenseBusAddress; /* Bytes 8-15 */ - DAC960_V2_PhysicalDevice_T PhysicalDevice; /* Bytes 16-18 */ - DAC960_V2_CommandTimeout_T CommandTimeout; /* Byte 19 */ - unsigned char RequestSenseSize; /* Byte 20 */ - unsigned char IOCTL_Opcode; /* Byte 21 */ - unsigned char Reserved[10]; /* Bytes 22-31 */ - DAC960_V2_DataTransferMemoryAddress_T - DataTransferMemoryAddress; /* Bytes 32-63 */ - } PhysicalDeviceInfo; - struct { - DAC960_V2_CommandIdentifier_T CommandIdentifier; /* Bytes 0-1 */ - DAC960_V2_CommandOpcode_T CommandOpcode; /* Byte 2 */ - DAC960_V2_CommandControlBits_T CommandControlBits; /* Byte 3 */ - DAC960_ByteCount32_T DataTransferSize:24; /* Bytes 4-6 */ - unsigned char DataTransferPageNumber; /* Byte 7 */ - DAC960_BusAddress64_T RequestSenseBusAddress; /* Bytes 8-15 */ - unsigned short EventSequenceNumberHigh16; /* Bytes 16-17 */ - unsigned char ControllerNumber; /* Byte 18 */ - DAC960_V2_CommandTimeout_T CommandTimeout; /* Byte 19 */ - unsigned char RequestSenseSize; /* Byte 20 */ - unsigned char IOCTL_Opcode; /* Byte 21 */ - unsigned short EventSequenceNumberLow16; /* Bytes 22-23 */ - unsigned char Reserved[8]; /* Bytes 24-31 */ - DAC960_V2_DataTransferMemoryAddress_T - DataTransferMemoryAddress; /* Bytes 32-63 */ - } GetEvent; - struct { - DAC960_V2_CommandIdentifier_T CommandIdentifier; /* Bytes 0-1 */ - DAC960_V2_CommandOpcode_T CommandOpcode; /* Byte 2 */ - DAC960_V2_CommandControlBits_T CommandControlBits; /* Byte 3 */ - DAC960_ByteCount32_T DataTransferSize:24; /* Bytes 4-6 */ - unsigned char DataTransferPageNumber; /* Byte 7 */ - DAC960_BusAddress64_T RequestSenseBusAddress; /* Bytes 8-15 */ - DAC960_V2_LogicalDevice_T LogicalDevice; /* Bytes 16-18 */ - DAC960_V2_CommandTimeout_T CommandTimeout; /* Byte 19 */ - unsigned char RequestSenseSize; /* Byte 20 */ - unsigned char IOCTL_Opcode; /* Byte 21 */ - union { - DAC960_V2_LogicalDeviceState_T LogicalDeviceState; - DAC960_V2_PhysicalDeviceState_T PhysicalDeviceState; - } DeviceState; /* Byte 22 */ - unsigned char Reserved[9]; /* Bytes 23-31 */ - DAC960_V2_DataTransferMemoryAddress_T - DataTransferMemoryAddress; /* Bytes 32-63 */ - } SetDeviceState; - struct { - DAC960_V2_CommandIdentifier_T CommandIdentifier; /* Bytes 0-1 */ - DAC960_V2_CommandOpcode_T CommandOpcode; /* Byte 2 */ - DAC960_V2_CommandControlBits_T CommandControlBits; /* Byte 3 */ - DAC960_ByteCount32_T DataTransferSize:24; /* Bytes 4-6 */ - unsigned char DataTransferPageNumber; /* Byte 7 */ - DAC960_BusAddress64_T RequestSenseBusAddress; /* Bytes 8-15 */ - DAC960_V2_LogicalDevice_T LogicalDevice; /* Bytes 16-18 */ - DAC960_V2_CommandTimeout_T CommandTimeout; /* Byte 19 */ - unsigned char RequestSenseSize; /* Byte 20 */ - unsigned char IOCTL_Opcode; /* Byte 21 */ - bool RestoreConsistency:1; /* Byte 22 Bit 0 */ - bool InitializedAreaOnly:1; /* Byte 22 Bit 1 */ - unsigned char :6; /* Byte 22 Bits 2-7 */ - unsigned char Reserved[9]; /* Bytes 23-31 */ - DAC960_V2_DataTransferMemoryAddress_T - DataTransferMemoryAddress; /* Bytes 32-63 */ - } ConsistencyCheck; - struct { - DAC960_V2_CommandIdentifier_T CommandIdentifier; /* Bytes 0-1 */ - DAC960_V2_CommandOpcode_T CommandOpcode; /* Byte 2 */ - DAC960_V2_CommandControlBits_T CommandControlBits; /* Byte 3 */ - unsigned char FirstCommandMailboxSizeKB; /* Byte 4 */ - unsigned char FirstStatusMailboxSizeKB; /* Byte 5 */ - unsigned char SecondCommandMailboxSizeKB; /* Byte 6 */ - unsigned char SecondStatusMailboxSizeKB; /* Byte 7 */ - DAC960_BusAddress64_T RequestSenseBusAddress; /* Bytes 8-15 */ - unsigned int :24; /* Bytes 16-18 */ - DAC960_V2_CommandTimeout_T CommandTimeout; /* Byte 19 */ - unsigned char RequestSenseSize; /* Byte 20 */ - unsigned char IOCTL_Opcode; /* Byte 21 */ - unsigned char HealthStatusBufferSizeKB; /* Byte 22 */ - unsigned char :8; /* Byte 23 */ - DAC960_BusAddress64_T HealthStatusBufferBusAddress; /* Bytes 24-31 */ - DAC960_BusAddress64_T FirstCommandMailboxBusAddress; /* Bytes 32-39 */ - DAC960_BusAddress64_T FirstStatusMailboxBusAddress; /* Bytes 40-47 */ - DAC960_BusAddress64_T SecondCommandMailboxBusAddress; /* Bytes 48-55 */ - DAC960_BusAddress64_T SecondStatusMailboxBusAddress; /* Bytes 56-63 */ - } SetMemoryMailbox; - struct { - DAC960_V2_CommandIdentifier_T CommandIdentifier; /* Bytes 0-1 */ - DAC960_V2_CommandOpcode_T CommandOpcode; /* Byte 2 */ - DAC960_V2_CommandControlBits_T CommandControlBits; /* Byte 3 */ - DAC960_ByteCount32_T DataTransferSize:24; /* Bytes 4-6 */ - unsigned char DataTransferPageNumber; /* Byte 7 */ - DAC960_BusAddress64_T RequestSenseBusAddress; /* Bytes 8-15 */ - DAC960_V2_PhysicalDevice_T PhysicalDevice; /* Bytes 16-18 */ - DAC960_V2_CommandTimeout_T CommandTimeout; /* Byte 19 */ - unsigned char RequestSenseSize; /* Byte 20 */ - unsigned char IOCTL_Opcode; /* Byte 21 */ - DAC960_V2_OperationDevice_T OperationDevice; /* Byte 22 */ - unsigned char Reserved[9]; /* Bytes 23-31 */ - DAC960_V2_DataTransferMemoryAddress_T - DataTransferMemoryAddress; /* Bytes 32-63 */ - } DeviceOperation; -} -DAC960_V2_CommandMailbox_T; - - -/* - Define the DAC960 Driver IOCTL requests. -*/ - -#define DAC960_IOCTL_GET_CONTROLLER_COUNT 0xDAC001 -#define DAC960_IOCTL_GET_CONTROLLER_INFO 0xDAC002 -#define DAC960_IOCTL_V1_EXECUTE_COMMAND 0xDAC003 -#define DAC960_IOCTL_V2_EXECUTE_COMMAND 0xDAC004 -#define DAC960_IOCTL_V2_GET_HEALTH_STATUS 0xDAC005 - - -/* - Define the DAC960_IOCTL_GET_CONTROLLER_INFO reply structure. -*/ - -typedef struct DAC960_ControllerInfo -{ - unsigned char ControllerNumber; - unsigned char FirmwareType; - unsigned char Channels; - unsigned char Targets; - unsigned char PCI_Bus; - unsigned char PCI_Device; - unsigned char PCI_Function; - unsigned char IRQ_Channel; - DAC960_PCI_Address_T PCI_Address; - unsigned char ModelName[20]; - unsigned char FirmwareVersion[12]; -} -DAC960_ControllerInfo_T; - - -/* - Define the User Mode DAC960_IOCTL_V1_EXECUTE_COMMAND request structure. -*/ - -typedef struct DAC960_V1_UserCommand -{ - unsigned char ControllerNumber; - DAC960_V1_CommandMailbox_T CommandMailbox; - int DataTransferLength; - void __user *DataTransferBuffer; - DAC960_V1_DCDB_T __user *DCDB; -} -DAC960_V1_UserCommand_T; - - -/* - Define the Kernel Mode DAC960_IOCTL_V1_EXECUTE_COMMAND request structure. -*/ - -typedef struct DAC960_V1_KernelCommand -{ - unsigned char ControllerNumber; - DAC960_V1_CommandMailbox_T CommandMailbox; - int DataTransferLength; - void *DataTransferBuffer; - DAC960_V1_DCDB_T *DCDB; - DAC960_V1_CommandStatus_T CommandStatus; - void (*CompletionFunction)(struct DAC960_V1_KernelCommand *); - void *CompletionData; -} -DAC960_V1_KernelCommand_T; - - -/* - Define the User Mode DAC960_IOCTL_V2_EXECUTE_COMMAND request structure. -*/ - -typedef struct DAC960_V2_UserCommand -{ - unsigned char ControllerNumber; - DAC960_V2_CommandMailbox_T CommandMailbox; - int DataTransferLength; - int RequestSenseLength; - void __user *DataTransferBuffer; - void __user *RequestSenseBuffer; -} -DAC960_V2_UserCommand_T; - - -/* - Define the Kernel Mode DAC960_IOCTL_V2_EXECUTE_COMMAND request structure. -*/ - -typedef struct DAC960_V2_KernelCommand -{ - unsigned char ControllerNumber; - DAC960_V2_CommandMailbox_T CommandMailbox; - int DataTransferLength; - int RequestSenseLength; - void *DataTransferBuffer; - void *RequestSenseBuffer; - DAC960_V2_CommandStatus_T CommandStatus; - void (*CompletionFunction)(struct DAC960_V2_KernelCommand *); - void *CompletionData; -} -DAC960_V2_KernelCommand_T; - - -/* - Define the User Mode DAC960_IOCTL_V2_GET_HEALTH_STATUS request structure. -*/ - -typedef struct DAC960_V2_GetHealthStatus -{ - unsigned char ControllerNumber; - DAC960_V2_HealthStatusBuffer_T __user *HealthStatusBuffer; -} -DAC960_V2_GetHealthStatus_T; - - -/* - Import the Kernel Mode IOCTL interface. -*/ - -extern int DAC960_KernelIOCTL(unsigned int Request, void *Argument); - - -/* - DAC960_DriverVersion protects the private portion of this file. -*/ - -#ifdef DAC960_DriverVersion - - -/* - Define the maximum Driver Queue Depth and Controller Queue Depth supported - by DAC960 V1 and V2 Firmware Controllers. -*/ - -#define DAC960_MaxDriverQueueDepth 511 -#define DAC960_MaxControllerQueueDepth 512 - - -/* - Define the maximum number of Scatter/Gather Segments supported for any - DAC960 V1 and V2 Firmware controller. -*/ - -#define DAC960_V1_ScatterGatherLimit 33 -#define DAC960_V2_ScatterGatherLimit 128 - - -/* - Define the number of Command Mailboxes and Status Mailboxes used by the - DAC960 V1 and V2 Firmware Memory Mailbox Interface. -*/ - -#define DAC960_V1_CommandMailboxCount 256 -#define DAC960_V1_StatusMailboxCount 1024 -#define DAC960_V2_CommandMailboxCount 512 -#define DAC960_V2_StatusMailboxCount 512 - - -/* - Define the DAC960 Controller Monitoring Timer Interval. -*/ - -#define DAC960_MonitoringTimerInterval (10 * HZ) - - -/* - Define the DAC960 Controller Secondary Monitoring Interval. -*/ - -#define DAC960_SecondaryMonitoringInterval (60 * HZ) - - -/* - Define the DAC960 Controller Health Status Monitoring Interval. -*/ - -#define DAC960_HealthStatusMonitoringInterval (1 * HZ) - - -/* - Define the DAC960 Controller Progress Reporting Interval. -*/ - -#define DAC960_ProgressReportingInterval (60 * HZ) - - -/* - Define the maximum number of Partitions allowed for each Logical Drive. -*/ - -#define DAC960_MaxPartitions 8 -#define DAC960_MaxPartitionsBits 3 - -/* - Define the DAC960 Controller fixed Block Size and Block Size Bits. -*/ - -#define DAC960_BlockSize 512 -#define DAC960_BlockSizeBits 9 - - -/* - Define the number of Command structures that should be allocated as a - group to optimize kernel memory allocation. -*/ - -#define DAC960_V1_CommandAllocationGroupSize 11 -#define DAC960_V2_CommandAllocationGroupSize 29 - - -/* - Define the Controller Line Buffer, Progress Buffer, User Message, and - Initial Status Buffer sizes. -*/ - -#define DAC960_LineBufferSize 100 -#define DAC960_ProgressBufferSize 200 -#define DAC960_UserMessageSize 200 -#define DAC960_InitialStatusBufferSize (8192-32) - - -/* - Define the DAC960 Controller Firmware Types. -*/ - -typedef enum -{ - DAC960_V1_Controller = 1, - DAC960_V2_Controller = 2 -} -DAC960_FirmwareType_T; - - -/* - Define the DAC960 Controller Hardware Types. -*/ - -typedef enum -{ - DAC960_BA_Controller = 1, /* eXtremeRAID 2000 */ - DAC960_LP_Controller = 2, /* AcceleRAID 352 */ - DAC960_LA_Controller = 3, /* DAC1164P */ - DAC960_PG_Controller = 4, /* DAC960PTL/PJ/PG */ - DAC960_PD_Controller = 5, /* DAC960PU/PD/PL/P */ - DAC960_P_Controller = 6, /* DAC960PU/PD/PL/P */ - DAC960_GEM_Controller = 7, /* AcceleRAID 4/5/600 */ -} -DAC960_HardwareType_T; - - -/* - Define the Driver Message Levels. -*/ - -typedef enum DAC960_MessageLevel -{ - DAC960_AnnounceLevel = 0, - DAC960_InfoLevel = 1, - DAC960_NoticeLevel = 2, - DAC960_WarningLevel = 3, - DAC960_ErrorLevel = 4, - DAC960_ProgressLevel = 5, - DAC960_CriticalLevel = 6, - DAC960_UserCriticalLevel = 7 -} -DAC960_MessageLevel_T; - -static char - *DAC960_MessageLevelMap[] = - { KERN_NOTICE, KERN_NOTICE, KERN_NOTICE, KERN_WARNING, - KERN_ERR, KERN_CRIT, KERN_CRIT, KERN_CRIT }; - - -/* - Define Driver Message macros. -*/ - -#define DAC960_Announce(Format, Arguments...) \ - DAC960_Message(DAC960_AnnounceLevel, Format, ##Arguments) - -#define DAC960_Info(Format, Arguments...) \ - DAC960_Message(DAC960_InfoLevel, Format, ##Arguments) - -#define DAC960_Notice(Format, Arguments...) \ - DAC960_Message(DAC960_NoticeLevel, Format, ##Arguments) - -#define DAC960_Warning(Format, Arguments...) \ - DAC960_Message(DAC960_WarningLevel, Format, ##Arguments) - -#define DAC960_Error(Format, Arguments...) \ - DAC960_Message(DAC960_ErrorLevel, Format, ##Arguments) - -#define DAC960_Progress(Format, Arguments...) \ - DAC960_Message(DAC960_ProgressLevel, Format, ##Arguments) - -#define DAC960_Critical(Format, Arguments...) \ - DAC960_Message(DAC960_CriticalLevel, Format, ##Arguments) - -#define DAC960_UserCritical(Format, Arguments...) \ - DAC960_Message(DAC960_UserCriticalLevel, Format, ##Arguments) - - -struct DAC960_privdata { - DAC960_HardwareType_T HardwareType; - DAC960_FirmwareType_T FirmwareType; - irq_handler_t InterruptHandler; - unsigned int MemoryWindowSize; -}; - - -/* - Define the DAC960 V1 Firmware Controller Status Mailbox structure. -*/ - -typedef union DAC960_V1_StatusMailbox -{ - unsigned int Word; /* Word 0 */ - struct { - DAC960_V1_CommandIdentifier_T CommandIdentifier; /* Byte 0 */ - unsigned char :7; /* Byte 1 Bits 0-6 */ - bool Valid:1; /* Byte 1 Bit 7 */ - DAC960_V1_CommandStatus_T CommandStatus; /* Bytes 2-3 */ - } Fields; -} -DAC960_V1_StatusMailbox_T; - - -/* - Define the DAC960 V2 Firmware Controller Status Mailbox structure. -*/ - -typedef union DAC960_V2_StatusMailbox -{ - unsigned int Words[2]; /* Words 0-1 */ - struct { - DAC960_V2_CommandIdentifier_T CommandIdentifier; /* Bytes 0-1 */ - DAC960_V2_CommandStatus_T CommandStatus; /* Byte 2 */ - unsigned char RequestSenseLength; /* Byte 3 */ - int DataTransferResidue; /* Bytes 4-7 */ - } Fields; -} -DAC960_V2_StatusMailbox_T; - - -/* - Define the DAC960 Driver Command Types. -*/ - -typedef enum -{ - DAC960_ReadCommand = 1, - DAC960_WriteCommand = 2, - DAC960_ReadRetryCommand = 3, - DAC960_WriteRetryCommand = 4, - DAC960_MonitoringCommand = 5, - DAC960_ImmediateCommand = 6, - DAC960_QueuedCommand = 7 -} -DAC960_CommandType_T; - - -/* - Define the DAC960 Driver Command structure. -*/ - -typedef struct DAC960_Command -{ - int CommandIdentifier; - DAC960_CommandType_T CommandType; - struct DAC960_Controller *Controller; - struct DAC960_Command *Next; - struct completion *Completion; - unsigned int LogicalDriveNumber; - unsigned int BlockNumber; - unsigned int BlockCount; - unsigned int SegmentCount; - int DmaDirection; - struct scatterlist *cmd_sglist; - struct request *Request; - union { - struct { - DAC960_V1_CommandMailbox_T CommandMailbox; - DAC960_V1_KernelCommand_T *KernelCommand; - DAC960_V1_CommandStatus_T CommandStatus; - DAC960_V1_ScatterGatherSegment_T *ScatterGatherList; - dma_addr_t ScatterGatherListDMA; - struct scatterlist ScatterList[DAC960_V1_ScatterGatherLimit]; - unsigned int EndMarker[0]; - } V1; - struct { - DAC960_V2_CommandMailbox_T CommandMailbox; - DAC960_V2_KernelCommand_T *KernelCommand; - DAC960_V2_CommandStatus_T CommandStatus; - unsigned char RequestSenseLength; - int DataTransferResidue; - DAC960_V2_ScatterGatherSegment_T *ScatterGatherList; - dma_addr_t ScatterGatherListDMA; - DAC960_SCSI_RequestSense_T *RequestSense; - dma_addr_t RequestSenseDMA; - struct scatterlist ScatterList[DAC960_V2_ScatterGatherLimit]; - unsigned int EndMarker[0]; - } V2; - } FW; -} -DAC960_Command_T; - - -/* - Define the DAC960 Driver Controller structure. -*/ - -typedef struct DAC960_Controller -{ - void __iomem *BaseAddress; - void __iomem *MemoryMappedAddress; - DAC960_FirmwareType_T FirmwareType; - DAC960_HardwareType_T HardwareType; - DAC960_IO_Address_T IO_Address; - DAC960_PCI_Address_T PCI_Address; - struct pci_dev *PCIDevice; - unsigned char ControllerNumber; - unsigned char ControllerName[4]; - unsigned char ModelName[20]; - unsigned char FullModelName[28]; - unsigned char FirmwareVersion[12]; - unsigned char Bus; - unsigned char Device; - unsigned char Function; - unsigned char IRQ_Channel; - unsigned char Channels; - unsigned char Targets; - unsigned char MemorySize; - unsigned char LogicalDriveCount; - unsigned short CommandAllocationGroupSize; - unsigned short ControllerQueueDepth; - unsigned short DriverQueueDepth; - unsigned short MaxBlocksPerCommand; - unsigned short ControllerScatterGatherLimit; - unsigned short DriverScatterGatherLimit; - unsigned int CombinedStatusBufferLength; - unsigned int InitialStatusLength; - unsigned int CurrentStatusLength; - unsigned int ProgressBufferLength; - unsigned int UserStatusLength; - struct dma_loaf DmaPages; - unsigned long MonitoringTimerCount; - unsigned long PrimaryMonitoringTime; - unsigned long SecondaryMonitoringTime; - unsigned long ShutdownMonitoringTimer; - unsigned long LastProgressReportTime; - unsigned long LastCurrentStatusTime; - bool ControllerInitialized; - bool MonitoringCommandDeferred; - bool EphemeralProgressMessage; - bool DriveSpinUpMessageDisplayed; - bool MonitoringAlertMode; - bool SuppressEnclosureMessages; - struct timer_list MonitoringTimer; - struct gendisk *disks[DAC960_MaxLogicalDrives]; - struct dma_pool *ScatterGatherPool; - DAC960_Command_T *FreeCommands; - unsigned char *CombinedStatusBuffer; - unsigned char *CurrentStatusBuffer; - struct request_queue *RequestQueue[DAC960_MaxLogicalDrives]; - int req_q_index; - spinlock_t queue_lock; - wait_queue_head_t CommandWaitQueue; - wait_queue_head_t HealthStatusWaitQueue; - DAC960_Command_T InitialCommand; - DAC960_Command_T *Commands[DAC960_MaxDriverQueueDepth]; - struct proc_dir_entry *ControllerProcEntry; - bool LogicalDriveInitiallyAccessible[DAC960_MaxLogicalDrives]; - void (*QueueCommand)(DAC960_Command_T *Command); - bool (*ReadControllerConfiguration)(struct DAC960_Controller *); - bool (*ReadDeviceConfiguration)(struct DAC960_Controller *); - bool (*ReportDeviceConfiguration)(struct DAC960_Controller *); - void (*QueueReadWriteCommand)(DAC960_Command_T *Command); - union { - struct { - unsigned char GeometryTranslationHeads; - unsigned char GeometryTranslationSectors; - unsigned char PendingRebuildFlag; - unsigned short StripeSize; - unsigned short SegmentSize; - unsigned short NewEventLogSequenceNumber; - unsigned short OldEventLogSequenceNumber; - unsigned short DeviceStateChannel; - unsigned short DeviceStateTargetID; - bool DualModeMemoryMailboxInterface; - bool BackgroundInitializationStatusSupported; - bool SAFTE_EnclosureManagementEnabled; - bool NeedLogicalDriveInformation; - bool NeedErrorTableInformation; - bool NeedDeviceStateInformation; - bool NeedDeviceInquiryInformation; - bool NeedDeviceSerialNumberInformation; - bool NeedRebuildProgress; - bool NeedConsistencyCheckProgress; - bool NeedBackgroundInitializationStatus; - bool StartDeviceStateScan; - bool RebuildProgressFirst; - bool RebuildFlagPending; - bool RebuildStatusPending; - - dma_addr_t FirstCommandMailboxDMA; - DAC960_V1_CommandMailbox_T *FirstCommandMailbox; - DAC960_V1_CommandMailbox_T *LastCommandMailbox; - DAC960_V1_CommandMailbox_T *NextCommandMailbox; - DAC960_V1_CommandMailbox_T *PreviousCommandMailbox1; - DAC960_V1_CommandMailbox_T *PreviousCommandMailbox2; - - dma_addr_t FirstStatusMailboxDMA; - DAC960_V1_StatusMailbox_T *FirstStatusMailbox; - DAC960_V1_StatusMailbox_T *LastStatusMailbox; - DAC960_V1_StatusMailbox_T *NextStatusMailbox; - - DAC960_V1_DCDB_T *MonitoringDCDB; - dma_addr_t MonitoringDCDB_DMA; - - DAC960_V1_Enquiry_T Enquiry; - DAC960_V1_Enquiry_T *NewEnquiry; - dma_addr_t NewEnquiryDMA; - - DAC960_V1_ErrorTable_T ErrorTable; - DAC960_V1_ErrorTable_T *NewErrorTable; - dma_addr_t NewErrorTableDMA; - - DAC960_V1_EventLogEntry_T *EventLogEntry; - dma_addr_t EventLogEntryDMA; - - DAC960_V1_RebuildProgress_T *RebuildProgress; - dma_addr_t RebuildProgressDMA; - DAC960_V1_CommandStatus_T LastRebuildStatus; - DAC960_V1_CommandStatus_T PendingRebuildStatus; - - DAC960_V1_LogicalDriveInformationArray_T LogicalDriveInformation; - DAC960_V1_LogicalDriveInformationArray_T *NewLogicalDriveInformation; - dma_addr_t NewLogicalDriveInformationDMA; - - DAC960_V1_BackgroundInitializationStatus_T - *BackgroundInitializationStatus; - dma_addr_t BackgroundInitializationStatusDMA; - DAC960_V1_BackgroundInitializationStatus_T - LastBackgroundInitializationStatus; - - DAC960_V1_DeviceState_T - DeviceState[DAC960_V1_MaxChannels][DAC960_V1_MaxTargets]; - DAC960_V1_DeviceState_T *NewDeviceState; - dma_addr_t NewDeviceStateDMA; - - DAC960_SCSI_Inquiry_T - InquiryStandardData[DAC960_V1_MaxChannels][DAC960_V1_MaxTargets]; - DAC960_SCSI_Inquiry_T *NewInquiryStandardData; - dma_addr_t NewInquiryStandardDataDMA; - - DAC960_SCSI_Inquiry_UnitSerialNumber_T - InquiryUnitSerialNumber[DAC960_V1_MaxChannels][DAC960_V1_MaxTargets]; - DAC960_SCSI_Inquiry_UnitSerialNumber_T *NewInquiryUnitSerialNumber; - dma_addr_t NewInquiryUnitSerialNumberDMA; - - int DeviceResetCount[DAC960_V1_MaxChannels][DAC960_V1_MaxTargets]; - bool DirectCommandActive[DAC960_V1_MaxChannels][DAC960_V1_MaxTargets]; - } V1; - struct { - unsigned int StatusChangeCounter; - unsigned int NextEventSequenceNumber; - unsigned int PhysicalDeviceIndex; - bool NeedLogicalDeviceInformation; - bool NeedPhysicalDeviceInformation; - bool NeedDeviceSerialNumberInformation; - bool StartLogicalDeviceInformationScan; - bool StartPhysicalDeviceInformationScan; - struct dma_pool *RequestSensePool; - - dma_addr_t FirstCommandMailboxDMA; - DAC960_V2_CommandMailbox_T *FirstCommandMailbox; - DAC960_V2_CommandMailbox_T *LastCommandMailbox; - DAC960_V2_CommandMailbox_T *NextCommandMailbox; - DAC960_V2_CommandMailbox_T *PreviousCommandMailbox1; - DAC960_V2_CommandMailbox_T *PreviousCommandMailbox2; - - dma_addr_t FirstStatusMailboxDMA; - DAC960_V2_StatusMailbox_T *FirstStatusMailbox; - DAC960_V2_StatusMailbox_T *LastStatusMailbox; - DAC960_V2_StatusMailbox_T *NextStatusMailbox; - - dma_addr_t HealthStatusBufferDMA; - DAC960_V2_HealthStatusBuffer_T *HealthStatusBuffer; - - DAC960_V2_ControllerInfo_T ControllerInformation; - DAC960_V2_ControllerInfo_T *NewControllerInformation; - dma_addr_t NewControllerInformationDMA; - - DAC960_V2_LogicalDeviceInfo_T - *LogicalDeviceInformation[DAC960_MaxLogicalDrives]; - DAC960_V2_LogicalDeviceInfo_T *NewLogicalDeviceInformation; - dma_addr_t NewLogicalDeviceInformationDMA; - - DAC960_V2_PhysicalDeviceInfo_T - *PhysicalDeviceInformation[DAC960_V2_MaxPhysicalDevices]; - DAC960_V2_PhysicalDeviceInfo_T *NewPhysicalDeviceInformation; - dma_addr_t NewPhysicalDeviceInformationDMA; - - DAC960_SCSI_Inquiry_UnitSerialNumber_T *NewInquiryUnitSerialNumber; - dma_addr_t NewInquiryUnitSerialNumberDMA; - DAC960_SCSI_Inquiry_UnitSerialNumber_T - *InquiryUnitSerialNumber[DAC960_V2_MaxPhysicalDevices]; - - DAC960_V2_Event_T *Event; - dma_addr_t EventDMA; - - DAC960_V2_PhysicalToLogicalDevice_T *PhysicalToLogicalDevice; - dma_addr_t PhysicalToLogicalDeviceDMA; - - DAC960_V2_PhysicalDevice_T - LogicalDriveToVirtualDevice[DAC960_MaxLogicalDrives]; - bool LogicalDriveFoundDuringScan[DAC960_MaxLogicalDrives]; - } V2; - } FW; - unsigned char ProgressBuffer[DAC960_ProgressBufferSize]; - unsigned char UserStatusBuffer[DAC960_UserMessageSize]; -} -DAC960_Controller_T; - - -/* - Simplify access to Firmware Version Dependent Data Structure Components - and Functions. -*/ - -#define V1 FW.V1 -#define V2 FW.V2 -#define DAC960_QueueCommand(Command) \ - (Controller->QueueCommand)(Command) -#define DAC960_ReadControllerConfiguration(Controller) \ - (Controller->ReadControllerConfiguration)(Controller) -#define DAC960_ReadDeviceConfiguration(Controller) \ - (Controller->ReadDeviceConfiguration)(Controller) -#define DAC960_ReportDeviceConfiguration(Controller) \ - (Controller->ReportDeviceConfiguration)(Controller) -#define DAC960_QueueReadWriteCommand(Command) \ - (Controller->QueueReadWriteCommand)(Command) - -/* - * dma_addr_writeql is provided to write dma_addr_t types - * to a 64-bit pci address space register. The controller - * will accept having the register written as two 32-bit - * values. - * - * In HIGHMEM kernels, dma_addr_t is a 64-bit value. - * without HIGHMEM, dma_addr_t is a 32-bit value. - * - * The compiler should always fix up the assignment - * to u.wq appropriately, depending upon the size of - * dma_addr_t. - */ -static inline -void dma_addr_writeql(dma_addr_t addr, void __iomem *write_address) -{ - union { - u64 wq; - uint wl[2]; - } u; - - u.wq = addr; - - writel(u.wl[0], write_address); - writel(u.wl[1], write_address + 4); -} - -/* - Define the DAC960 GEM Series Controller Interface Register Offsets. - */ - -#define DAC960_GEM_RegisterWindowSize 0x600 - -typedef enum -{ - DAC960_GEM_InboundDoorBellRegisterReadSetOffset = 0x214, - DAC960_GEM_InboundDoorBellRegisterClearOffset = 0x218, - DAC960_GEM_OutboundDoorBellRegisterReadSetOffset = 0x224, - DAC960_GEM_OutboundDoorBellRegisterClearOffset = 0x228, - DAC960_GEM_InterruptStatusRegisterOffset = 0x208, - DAC960_GEM_InterruptMaskRegisterReadSetOffset = 0x22C, - DAC960_GEM_InterruptMaskRegisterClearOffset = 0x230, - DAC960_GEM_CommandMailboxBusAddressOffset = 0x510, - DAC960_GEM_CommandStatusOffset = 0x518, - DAC960_GEM_ErrorStatusRegisterReadSetOffset = 0x224, - DAC960_GEM_ErrorStatusRegisterClearOffset = 0x228, -} -DAC960_GEM_RegisterOffsets_T; - -/* - Define the structure of the DAC960 GEM Series Inbound Door Bell - */ - -typedef union DAC960_GEM_InboundDoorBellRegister -{ - unsigned int All; - struct { - unsigned int :24; - bool HardwareMailboxNewCommand:1; - bool AcknowledgeHardwareMailboxStatus:1; - bool GenerateInterrupt:1; - bool ControllerReset:1; - bool MemoryMailboxNewCommand:1; - unsigned int :3; - } Write; - struct { - unsigned int :24; - bool HardwareMailboxFull:1; - bool InitializationInProgress:1; - unsigned int :6; - } Read; -} -DAC960_GEM_InboundDoorBellRegister_T; - -/* - Define the structure of the DAC960 GEM Series Outbound Door Bell Register. - */ -typedef union DAC960_GEM_OutboundDoorBellRegister -{ - unsigned int All; - struct { - unsigned int :24; - bool AcknowledgeHardwareMailboxInterrupt:1; - bool AcknowledgeMemoryMailboxInterrupt:1; - unsigned int :6; - } Write; - struct { - unsigned int :24; - bool HardwareMailboxStatusAvailable:1; - bool MemoryMailboxStatusAvailable:1; - unsigned int :6; - } Read; -} -DAC960_GEM_OutboundDoorBellRegister_T; - -/* - Define the structure of the DAC960 GEM Series Interrupt Mask Register. - */ -typedef union DAC960_GEM_InterruptMaskRegister -{ - unsigned int All; - struct { - unsigned int :16; - unsigned int :8; - unsigned int HardwareMailboxInterrupt:1; - unsigned int MemoryMailboxInterrupt:1; - unsigned int :6; - } Bits; -} -DAC960_GEM_InterruptMaskRegister_T; - -/* - Define the structure of the DAC960 GEM Series Error Status Register. - */ - -typedef union DAC960_GEM_ErrorStatusRegister -{ - unsigned int All; - struct { - unsigned int :24; - unsigned int :5; - bool ErrorStatusPending:1; - unsigned int :2; - } Bits; -} -DAC960_GEM_ErrorStatusRegister_T; - -/* - Define inline functions to provide an abstraction for reading and writing the - DAC960 GEM Series Controller Interface Registers. -*/ - -static inline -void DAC960_GEM_HardwareMailboxNewCommand(void __iomem *ControllerBaseAddress) -{ - DAC960_GEM_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.HardwareMailboxNewCommand = true; - writel(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_GEM_InboundDoorBellRegisterReadSetOffset); -} - -static inline -void DAC960_GEM_AcknowledgeHardwareMailboxStatus(void __iomem *ControllerBaseAddress) -{ - DAC960_GEM_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.AcknowledgeHardwareMailboxStatus = true; - writel(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_GEM_InboundDoorBellRegisterClearOffset); -} - -static inline -void DAC960_GEM_GenerateInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_GEM_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.GenerateInterrupt = true; - writel(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_GEM_InboundDoorBellRegisterReadSetOffset); -} - -static inline -void DAC960_GEM_ControllerReset(void __iomem *ControllerBaseAddress) -{ - DAC960_GEM_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.ControllerReset = true; - writel(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_GEM_InboundDoorBellRegisterReadSetOffset); -} - -static inline -void DAC960_GEM_MemoryMailboxNewCommand(void __iomem *ControllerBaseAddress) -{ - DAC960_GEM_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.MemoryMailboxNewCommand = true; - writel(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_GEM_InboundDoorBellRegisterReadSetOffset); -} - -static inline -bool DAC960_GEM_HardwareMailboxFullP(void __iomem *ControllerBaseAddress) -{ - DAC960_GEM_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = - readl(ControllerBaseAddress + - DAC960_GEM_InboundDoorBellRegisterReadSetOffset); - return InboundDoorBellRegister.Read.HardwareMailboxFull; -} - -static inline -bool DAC960_GEM_InitializationInProgressP(void __iomem *ControllerBaseAddress) -{ - DAC960_GEM_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = - readl(ControllerBaseAddress + - DAC960_GEM_InboundDoorBellRegisterReadSetOffset); - return InboundDoorBellRegister.Read.InitializationInProgress; -} - -static inline -void DAC960_GEM_AcknowledgeHardwareMailboxInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_GEM_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeHardwareMailboxInterrupt = true; - writel(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_GEM_OutboundDoorBellRegisterClearOffset); -} - -static inline -void DAC960_GEM_AcknowledgeMemoryMailboxInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_GEM_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeMemoryMailboxInterrupt = true; - writel(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_GEM_OutboundDoorBellRegisterClearOffset); -} - -static inline -void DAC960_GEM_AcknowledgeInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_GEM_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeHardwareMailboxInterrupt = true; - OutboundDoorBellRegister.Write.AcknowledgeMemoryMailboxInterrupt = true; - writel(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_GEM_OutboundDoorBellRegisterClearOffset); -} - -static inline -bool DAC960_GEM_HardwareMailboxStatusAvailableP(void __iomem *ControllerBaseAddress) -{ - DAC960_GEM_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = - readl(ControllerBaseAddress + - DAC960_GEM_OutboundDoorBellRegisterReadSetOffset); - return OutboundDoorBellRegister.Read.HardwareMailboxStatusAvailable; -} - -static inline -bool DAC960_GEM_MemoryMailboxStatusAvailableP(void __iomem *ControllerBaseAddress) -{ - DAC960_GEM_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = - readl(ControllerBaseAddress + - DAC960_GEM_OutboundDoorBellRegisterReadSetOffset); - return OutboundDoorBellRegister.Read.MemoryMailboxStatusAvailable; -} - -static inline -void DAC960_GEM_EnableInterrupts(void __iomem *ControllerBaseAddress) -{ - DAC960_GEM_InterruptMaskRegister_T InterruptMaskRegister; - InterruptMaskRegister.All = 0; - InterruptMaskRegister.Bits.HardwareMailboxInterrupt = true; - InterruptMaskRegister.Bits.MemoryMailboxInterrupt = true; - writel(InterruptMaskRegister.All, - ControllerBaseAddress + DAC960_GEM_InterruptMaskRegisterClearOffset); -} - -static inline -void DAC960_GEM_DisableInterrupts(void __iomem *ControllerBaseAddress) -{ - DAC960_GEM_InterruptMaskRegister_T InterruptMaskRegister; - InterruptMaskRegister.All = 0; - InterruptMaskRegister.Bits.HardwareMailboxInterrupt = true; - InterruptMaskRegister.Bits.MemoryMailboxInterrupt = true; - writel(InterruptMaskRegister.All, - ControllerBaseAddress + DAC960_GEM_InterruptMaskRegisterReadSetOffset); -} - -static inline -bool DAC960_GEM_InterruptsEnabledP(void __iomem *ControllerBaseAddress) -{ - DAC960_GEM_InterruptMaskRegister_T InterruptMaskRegister; - InterruptMaskRegister.All = - readl(ControllerBaseAddress + - DAC960_GEM_InterruptMaskRegisterReadSetOffset); - return !(InterruptMaskRegister.Bits.HardwareMailboxInterrupt || - InterruptMaskRegister.Bits.MemoryMailboxInterrupt); -} - -static inline -void DAC960_GEM_WriteCommandMailbox(DAC960_V2_CommandMailbox_T - *MemoryCommandMailbox, - DAC960_V2_CommandMailbox_T - *CommandMailbox) -{ - memcpy(&MemoryCommandMailbox->Words[1], &CommandMailbox->Words[1], - sizeof(DAC960_V2_CommandMailbox_T) - sizeof(unsigned int)); - wmb(); - MemoryCommandMailbox->Words[0] = CommandMailbox->Words[0]; - mb(); -} - -static inline -void DAC960_GEM_WriteHardwareMailbox(void __iomem *ControllerBaseAddress, - dma_addr_t CommandMailboxDMA) -{ - dma_addr_writeql(CommandMailboxDMA, - ControllerBaseAddress + - DAC960_GEM_CommandMailboxBusAddressOffset); -} - -static inline DAC960_V2_CommandIdentifier_T -DAC960_GEM_ReadCommandIdentifier(void __iomem *ControllerBaseAddress) -{ - return readw(ControllerBaseAddress + DAC960_GEM_CommandStatusOffset); -} - -static inline DAC960_V2_CommandStatus_T -DAC960_GEM_ReadCommandStatus(void __iomem *ControllerBaseAddress) -{ - return readw(ControllerBaseAddress + DAC960_GEM_CommandStatusOffset + 2); -} - -static inline bool -DAC960_GEM_ReadErrorStatus(void __iomem *ControllerBaseAddress, - unsigned char *ErrorStatus, - unsigned char *Parameter0, - unsigned char *Parameter1) -{ - DAC960_GEM_ErrorStatusRegister_T ErrorStatusRegister; - ErrorStatusRegister.All = - readl(ControllerBaseAddress + DAC960_GEM_ErrorStatusRegisterReadSetOffset); - if (!ErrorStatusRegister.Bits.ErrorStatusPending) return false; - ErrorStatusRegister.Bits.ErrorStatusPending = false; - *ErrorStatus = ErrorStatusRegister.All; - *Parameter0 = - readb(ControllerBaseAddress + DAC960_GEM_CommandMailboxBusAddressOffset + 0); - *Parameter1 = - readb(ControllerBaseAddress + DAC960_GEM_CommandMailboxBusAddressOffset + 1); - writel(0x03000000, ControllerBaseAddress + - DAC960_GEM_ErrorStatusRegisterClearOffset); - return true; -} - -/* - Define the DAC960 BA Series Controller Interface Register Offsets. -*/ - -#define DAC960_BA_RegisterWindowSize 0x80 - -typedef enum -{ - DAC960_BA_InboundDoorBellRegisterOffset = 0x60, - DAC960_BA_OutboundDoorBellRegisterOffset = 0x61, - DAC960_BA_InterruptStatusRegisterOffset = 0x30, - DAC960_BA_InterruptMaskRegisterOffset = 0x34, - DAC960_BA_CommandMailboxBusAddressOffset = 0x50, - DAC960_BA_CommandStatusOffset = 0x58, - DAC960_BA_ErrorStatusRegisterOffset = 0x63 -} -DAC960_BA_RegisterOffsets_T; - - -/* - Define the structure of the DAC960 BA Series Inbound Door Bell Register. -*/ - -typedef union DAC960_BA_InboundDoorBellRegister -{ - unsigned char All; - struct { - bool HardwareMailboxNewCommand:1; /* Bit 0 */ - bool AcknowledgeHardwareMailboxStatus:1; /* Bit 1 */ - bool GenerateInterrupt:1; /* Bit 2 */ - bool ControllerReset:1; /* Bit 3 */ - bool MemoryMailboxNewCommand:1; /* Bit 4 */ - unsigned char :3; /* Bits 5-7 */ - } Write; - struct { - bool HardwareMailboxEmpty:1; /* Bit 0 */ - bool InitializationNotInProgress:1; /* Bit 1 */ - unsigned char :6; /* Bits 2-7 */ - } Read; -} -DAC960_BA_InboundDoorBellRegister_T; - - -/* - Define the structure of the DAC960 BA Series Outbound Door Bell Register. -*/ - -typedef union DAC960_BA_OutboundDoorBellRegister -{ - unsigned char All; - struct { - bool AcknowledgeHardwareMailboxInterrupt:1; /* Bit 0 */ - bool AcknowledgeMemoryMailboxInterrupt:1; /* Bit 1 */ - unsigned char :6; /* Bits 2-7 */ - } Write; - struct { - bool HardwareMailboxStatusAvailable:1; /* Bit 0 */ - bool MemoryMailboxStatusAvailable:1; /* Bit 1 */ - unsigned char :6; /* Bits 2-7 */ - } Read; -} -DAC960_BA_OutboundDoorBellRegister_T; - - -/* - Define the structure of the DAC960 BA Series Interrupt Mask Register. -*/ - -typedef union DAC960_BA_InterruptMaskRegister -{ - unsigned char All; - struct { - unsigned int :2; /* Bits 0-1 */ - bool DisableInterrupts:1; /* Bit 2 */ - bool DisableInterruptsI2O:1; /* Bit 3 */ - unsigned int :4; /* Bits 4-7 */ - } Bits; -} -DAC960_BA_InterruptMaskRegister_T; - - -/* - Define the structure of the DAC960 BA Series Error Status Register. -*/ - -typedef union DAC960_BA_ErrorStatusRegister -{ - unsigned char All; - struct { - unsigned int :2; /* Bits 0-1 */ - bool ErrorStatusPending:1; /* Bit 2 */ - unsigned int :5; /* Bits 3-7 */ - } Bits; -} -DAC960_BA_ErrorStatusRegister_T; - - -/* - Define inline functions to provide an abstraction for reading and writing the - DAC960 BA Series Controller Interface Registers. -*/ - -static inline -void DAC960_BA_HardwareMailboxNewCommand(void __iomem *ControllerBaseAddress) -{ - DAC960_BA_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.HardwareMailboxNewCommand = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_BA_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_BA_AcknowledgeHardwareMailboxStatus(void __iomem *ControllerBaseAddress) -{ - DAC960_BA_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.AcknowledgeHardwareMailboxStatus = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_BA_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_BA_GenerateInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_BA_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.GenerateInterrupt = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_BA_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_BA_ControllerReset(void __iomem *ControllerBaseAddress) -{ - DAC960_BA_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.ControllerReset = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_BA_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_BA_MemoryMailboxNewCommand(void __iomem *ControllerBaseAddress) -{ - DAC960_BA_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.MemoryMailboxNewCommand = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_BA_InboundDoorBellRegisterOffset); -} - -static inline -bool DAC960_BA_HardwareMailboxFullP(void __iomem *ControllerBaseAddress) -{ - DAC960_BA_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = - readb(ControllerBaseAddress + DAC960_BA_InboundDoorBellRegisterOffset); - return !InboundDoorBellRegister.Read.HardwareMailboxEmpty; -} - -static inline -bool DAC960_BA_InitializationInProgressP(void __iomem *ControllerBaseAddress) -{ - DAC960_BA_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = - readb(ControllerBaseAddress + DAC960_BA_InboundDoorBellRegisterOffset); - return !InboundDoorBellRegister.Read.InitializationNotInProgress; -} - -static inline -void DAC960_BA_AcknowledgeHardwareMailboxInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_BA_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeHardwareMailboxInterrupt = true; - writeb(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_BA_OutboundDoorBellRegisterOffset); -} - -static inline -void DAC960_BA_AcknowledgeMemoryMailboxInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_BA_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeMemoryMailboxInterrupt = true; - writeb(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_BA_OutboundDoorBellRegisterOffset); -} - -static inline -void DAC960_BA_AcknowledgeInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_BA_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeHardwareMailboxInterrupt = true; - OutboundDoorBellRegister.Write.AcknowledgeMemoryMailboxInterrupt = true; - writeb(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_BA_OutboundDoorBellRegisterOffset); -} - -static inline -bool DAC960_BA_HardwareMailboxStatusAvailableP(void __iomem *ControllerBaseAddress) -{ - DAC960_BA_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = - readb(ControllerBaseAddress + DAC960_BA_OutboundDoorBellRegisterOffset); - return OutboundDoorBellRegister.Read.HardwareMailboxStatusAvailable; -} - -static inline -bool DAC960_BA_MemoryMailboxStatusAvailableP(void __iomem *ControllerBaseAddress) -{ - DAC960_BA_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = - readb(ControllerBaseAddress + DAC960_BA_OutboundDoorBellRegisterOffset); - return OutboundDoorBellRegister.Read.MemoryMailboxStatusAvailable; -} - -static inline -void DAC960_BA_EnableInterrupts(void __iomem *ControllerBaseAddress) -{ - DAC960_BA_InterruptMaskRegister_T InterruptMaskRegister; - InterruptMaskRegister.All = 0xFF; - InterruptMaskRegister.Bits.DisableInterrupts = false; - InterruptMaskRegister.Bits.DisableInterruptsI2O = true; - writeb(InterruptMaskRegister.All, - ControllerBaseAddress + DAC960_BA_InterruptMaskRegisterOffset); -} - -static inline -void DAC960_BA_DisableInterrupts(void __iomem *ControllerBaseAddress) -{ - DAC960_BA_InterruptMaskRegister_T InterruptMaskRegister; - InterruptMaskRegister.All = 0xFF; - InterruptMaskRegister.Bits.DisableInterrupts = true; - InterruptMaskRegister.Bits.DisableInterruptsI2O = true; - writeb(InterruptMaskRegister.All, - ControllerBaseAddress + DAC960_BA_InterruptMaskRegisterOffset); -} - -static inline -bool DAC960_BA_InterruptsEnabledP(void __iomem *ControllerBaseAddress) -{ - DAC960_BA_InterruptMaskRegister_T InterruptMaskRegister; - InterruptMaskRegister.All = - readb(ControllerBaseAddress + DAC960_BA_InterruptMaskRegisterOffset); - return !InterruptMaskRegister.Bits.DisableInterrupts; -} - -static inline -void DAC960_BA_WriteCommandMailbox(DAC960_V2_CommandMailbox_T - *MemoryCommandMailbox, - DAC960_V2_CommandMailbox_T - *CommandMailbox) -{ - memcpy(&MemoryCommandMailbox->Words[1], &CommandMailbox->Words[1], - sizeof(DAC960_V2_CommandMailbox_T) - sizeof(unsigned int)); - wmb(); - MemoryCommandMailbox->Words[0] = CommandMailbox->Words[0]; - mb(); -} - - -static inline -void DAC960_BA_WriteHardwareMailbox(void __iomem *ControllerBaseAddress, - dma_addr_t CommandMailboxDMA) -{ - dma_addr_writeql(CommandMailboxDMA, - ControllerBaseAddress + - DAC960_BA_CommandMailboxBusAddressOffset); -} - -static inline DAC960_V2_CommandIdentifier_T -DAC960_BA_ReadCommandIdentifier(void __iomem *ControllerBaseAddress) -{ - return readw(ControllerBaseAddress + DAC960_BA_CommandStatusOffset); -} - -static inline DAC960_V2_CommandStatus_T -DAC960_BA_ReadCommandStatus(void __iomem *ControllerBaseAddress) -{ - return readw(ControllerBaseAddress + DAC960_BA_CommandStatusOffset + 2); -} - -static inline bool -DAC960_BA_ReadErrorStatus(void __iomem *ControllerBaseAddress, - unsigned char *ErrorStatus, - unsigned char *Parameter0, - unsigned char *Parameter1) -{ - DAC960_BA_ErrorStatusRegister_T ErrorStatusRegister; - ErrorStatusRegister.All = - readb(ControllerBaseAddress + DAC960_BA_ErrorStatusRegisterOffset); - if (!ErrorStatusRegister.Bits.ErrorStatusPending) return false; - ErrorStatusRegister.Bits.ErrorStatusPending = false; - *ErrorStatus = ErrorStatusRegister.All; - *Parameter0 = - readb(ControllerBaseAddress + DAC960_BA_CommandMailboxBusAddressOffset + 0); - *Parameter1 = - readb(ControllerBaseAddress + DAC960_BA_CommandMailboxBusAddressOffset + 1); - writeb(0xFF, ControllerBaseAddress + DAC960_BA_ErrorStatusRegisterOffset); - return true; -} - - -/* - Define the DAC960 LP Series Controller Interface Register Offsets. -*/ - -#define DAC960_LP_RegisterWindowSize 0x80 - -typedef enum -{ - DAC960_LP_InboundDoorBellRegisterOffset = 0x20, - DAC960_LP_OutboundDoorBellRegisterOffset = 0x2C, - DAC960_LP_InterruptStatusRegisterOffset = 0x30, - DAC960_LP_InterruptMaskRegisterOffset = 0x34, - DAC960_LP_CommandMailboxBusAddressOffset = 0x10, - DAC960_LP_CommandStatusOffset = 0x18, - DAC960_LP_ErrorStatusRegisterOffset = 0x2E -} -DAC960_LP_RegisterOffsets_T; - - -/* - Define the structure of the DAC960 LP Series Inbound Door Bell Register. -*/ - -typedef union DAC960_LP_InboundDoorBellRegister -{ - unsigned char All; - struct { - bool HardwareMailboxNewCommand:1; /* Bit 0 */ - bool AcknowledgeHardwareMailboxStatus:1; /* Bit 1 */ - bool GenerateInterrupt:1; /* Bit 2 */ - bool ControllerReset:1; /* Bit 3 */ - bool MemoryMailboxNewCommand:1; /* Bit 4 */ - unsigned char :3; /* Bits 5-7 */ - } Write; - struct { - bool HardwareMailboxFull:1; /* Bit 0 */ - bool InitializationInProgress:1; /* Bit 1 */ - unsigned char :6; /* Bits 2-7 */ - } Read; -} -DAC960_LP_InboundDoorBellRegister_T; - - -/* - Define the structure of the DAC960 LP Series Outbound Door Bell Register. -*/ - -typedef union DAC960_LP_OutboundDoorBellRegister -{ - unsigned char All; - struct { - bool AcknowledgeHardwareMailboxInterrupt:1; /* Bit 0 */ - bool AcknowledgeMemoryMailboxInterrupt:1; /* Bit 1 */ - unsigned char :6; /* Bits 2-7 */ - } Write; - struct { - bool HardwareMailboxStatusAvailable:1; /* Bit 0 */ - bool MemoryMailboxStatusAvailable:1; /* Bit 1 */ - unsigned char :6; /* Bits 2-7 */ - } Read; -} -DAC960_LP_OutboundDoorBellRegister_T; - - -/* - Define the structure of the DAC960 LP Series Interrupt Mask Register. -*/ - -typedef union DAC960_LP_InterruptMaskRegister -{ - unsigned char All; - struct { - unsigned int :2; /* Bits 0-1 */ - bool DisableInterrupts:1; /* Bit 2 */ - unsigned int :5; /* Bits 3-7 */ - } Bits; -} -DAC960_LP_InterruptMaskRegister_T; - - -/* - Define the structure of the DAC960 LP Series Error Status Register. -*/ - -typedef union DAC960_LP_ErrorStatusRegister -{ - unsigned char All; - struct { - unsigned int :2; /* Bits 0-1 */ - bool ErrorStatusPending:1; /* Bit 2 */ - unsigned int :5; /* Bits 3-7 */ - } Bits; -} -DAC960_LP_ErrorStatusRegister_T; - - -/* - Define inline functions to provide an abstraction for reading and writing the - DAC960 LP Series Controller Interface Registers. -*/ - -static inline -void DAC960_LP_HardwareMailboxNewCommand(void __iomem *ControllerBaseAddress) -{ - DAC960_LP_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.HardwareMailboxNewCommand = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LP_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_LP_AcknowledgeHardwareMailboxStatus(void __iomem *ControllerBaseAddress) -{ - DAC960_LP_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.AcknowledgeHardwareMailboxStatus = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LP_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_LP_GenerateInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_LP_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.GenerateInterrupt = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LP_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_LP_ControllerReset(void __iomem *ControllerBaseAddress) -{ - DAC960_LP_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.ControllerReset = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LP_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_LP_MemoryMailboxNewCommand(void __iomem *ControllerBaseAddress) -{ - DAC960_LP_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.MemoryMailboxNewCommand = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LP_InboundDoorBellRegisterOffset); -} - -static inline -bool DAC960_LP_HardwareMailboxFullP(void __iomem *ControllerBaseAddress) -{ - DAC960_LP_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = - readb(ControllerBaseAddress + DAC960_LP_InboundDoorBellRegisterOffset); - return InboundDoorBellRegister.Read.HardwareMailboxFull; -} - -static inline -bool DAC960_LP_InitializationInProgressP(void __iomem *ControllerBaseAddress) -{ - DAC960_LP_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = - readb(ControllerBaseAddress + DAC960_LP_InboundDoorBellRegisterOffset); - return InboundDoorBellRegister.Read.InitializationInProgress; -} - -static inline -void DAC960_LP_AcknowledgeHardwareMailboxInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_LP_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeHardwareMailboxInterrupt = true; - writeb(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LP_OutboundDoorBellRegisterOffset); -} - -static inline -void DAC960_LP_AcknowledgeMemoryMailboxInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_LP_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeMemoryMailboxInterrupt = true; - writeb(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LP_OutboundDoorBellRegisterOffset); -} - -static inline -void DAC960_LP_AcknowledgeInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_LP_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeHardwareMailboxInterrupt = true; - OutboundDoorBellRegister.Write.AcknowledgeMemoryMailboxInterrupt = true; - writeb(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LP_OutboundDoorBellRegisterOffset); -} - -static inline -bool DAC960_LP_HardwareMailboxStatusAvailableP(void __iomem *ControllerBaseAddress) -{ - DAC960_LP_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = - readb(ControllerBaseAddress + DAC960_LP_OutboundDoorBellRegisterOffset); - return OutboundDoorBellRegister.Read.HardwareMailboxStatusAvailable; -} - -static inline -bool DAC960_LP_MemoryMailboxStatusAvailableP(void __iomem *ControllerBaseAddress) -{ - DAC960_LP_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = - readb(ControllerBaseAddress + DAC960_LP_OutboundDoorBellRegisterOffset); - return OutboundDoorBellRegister.Read.MemoryMailboxStatusAvailable; -} - -static inline -void DAC960_LP_EnableInterrupts(void __iomem *ControllerBaseAddress) -{ - DAC960_LP_InterruptMaskRegister_T InterruptMaskRegister; - InterruptMaskRegister.All = 0xFF; - InterruptMaskRegister.Bits.DisableInterrupts = false; - writeb(InterruptMaskRegister.All, - ControllerBaseAddress + DAC960_LP_InterruptMaskRegisterOffset); -} - -static inline -void DAC960_LP_DisableInterrupts(void __iomem *ControllerBaseAddress) -{ - DAC960_LP_InterruptMaskRegister_T InterruptMaskRegister; - InterruptMaskRegister.All = 0xFF; - InterruptMaskRegister.Bits.DisableInterrupts = true; - writeb(InterruptMaskRegister.All, - ControllerBaseAddress + DAC960_LP_InterruptMaskRegisterOffset); -} - -static inline -bool DAC960_LP_InterruptsEnabledP(void __iomem *ControllerBaseAddress) -{ - DAC960_LP_InterruptMaskRegister_T InterruptMaskRegister; - InterruptMaskRegister.All = - readb(ControllerBaseAddress + DAC960_LP_InterruptMaskRegisterOffset); - return !InterruptMaskRegister.Bits.DisableInterrupts; -} - -static inline -void DAC960_LP_WriteCommandMailbox(DAC960_V2_CommandMailbox_T - *MemoryCommandMailbox, - DAC960_V2_CommandMailbox_T - *CommandMailbox) -{ - memcpy(&MemoryCommandMailbox->Words[1], &CommandMailbox->Words[1], - sizeof(DAC960_V2_CommandMailbox_T) - sizeof(unsigned int)); - wmb(); - MemoryCommandMailbox->Words[0] = CommandMailbox->Words[0]; - mb(); -} - -static inline -void DAC960_LP_WriteHardwareMailbox(void __iomem *ControllerBaseAddress, - dma_addr_t CommandMailboxDMA) -{ - dma_addr_writeql(CommandMailboxDMA, - ControllerBaseAddress + - DAC960_LP_CommandMailboxBusAddressOffset); -} - -static inline DAC960_V2_CommandIdentifier_T -DAC960_LP_ReadCommandIdentifier(void __iomem *ControllerBaseAddress) -{ - return readw(ControllerBaseAddress + DAC960_LP_CommandStatusOffset); -} - -static inline DAC960_V2_CommandStatus_T -DAC960_LP_ReadCommandStatus(void __iomem *ControllerBaseAddress) -{ - return readw(ControllerBaseAddress + DAC960_LP_CommandStatusOffset + 2); -} - -static inline bool -DAC960_LP_ReadErrorStatus(void __iomem *ControllerBaseAddress, - unsigned char *ErrorStatus, - unsigned char *Parameter0, - unsigned char *Parameter1) -{ - DAC960_LP_ErrorStatusRegister_T ErrorStatusRegister; - ErrorStatusRegister.All = - readb(ControllerBaseAddress + DAC960_LP_ErrorStatusRegisterOffset); - if (!ErrorStatusRegister.Bits.ErrorStatusPending) return false; - ErrorStatusRegister.Bits.ErrorStatusPending = false; - *ErrorStatus = ErrorStatusRegister.All; - *Parameter0 = - readb(ControllerBaseAddress + DAC960_LP_CommandMailboxBusAddressOffset + 0); - *Parameter1 = - readb(ControllerBaseAddress + DAC960_LP_CommandMailboxBusAddressOffset + 1); - writeb(0xFF, ControllerBaseAddress + DAC960_LP_ErrorStatusRegisterOffset); - return true; -} - - -/* - Define the DAC960 LA Series Controller Interface Register Offsets. -*/ - -#define DAC960_LA_RegisterWindowSize 0x80 - -typedef enum -{ - DAC960_LA_InboundDoorBellRegisterOffset = 0x60, - DAC960_LA_OutboundDoorBellRegisterOffset = 0x61, - DAC960_LA_InterruptMaskRegisterOffset = 0x34, - DAC960_LA_CommandOpcodeRegisterOffset = 0x50, - DAC960_LA_CommandIdentifierRegisterOffset = 0x51, - DAC960_LA_MailboxRegister2Offset = 0x52, - DAC960_LA_MailboxRegister3Offset = 0x53, - DAC960_LA_MailboxRegister4Offset = 0x54, - DAC960_LA_MailboxRegister5Offset = 0x55, - DAC960_LA_MailboxRegister6Offset = 0x56, - DAC960_LA_MailboxRegister7Offset = 0x57, - DAC960_LA_MailboxRegister8Offset = 0x58, - DAC960_LA_MailboxRegister9Offset = 0x59, - DAC960_LA_MailboxRegister10Offset = 0x5A, - DAC960_LA_MailboxRegister11Offset = 0x5B, - DAC960_LA_MailboxRegister12Offset = 0x5C, - DAC960_LA_StatusCommandIdentifierRegOffset = 0x5D, - DAC960_LA_StatusRegisterOffset = 0x5E, - DAC960_LA_ErrorStatusRegisterOffset = 0x63 -} -DAC960_LA_RegisterOffsets_T; - - -/* - Define the structure of the DAC960 LA Series Inbound Door Bell Register. -*/ - -typedef union DAC960_LA_InboundDoorBellRegister -{ - unsigned char All; - struct { - bool HardwareMailboxNewCommand:1; /* Bit 0 */ - bool AcknowledgeHardwareMailboxStatus:1; /* Bit 1 */ - bool GenerateInterrupt:1; /* Bit 2 */ - bool ControllerReset:1; /* Bit 3 */ - bool MemoryMailboxNewCommand:1; /* Bit 4 */ - unsigned char :3; /* Bits 5-7 */ - } Write; - struct { - bool HardwareMailboxEmpty:1; /* Bit 0 */ - bool InitializationNotInProgress:1; /* Bit 1 */ - unsigned char :6; /* Bits 2-7 */ - } Read; -} -DAC960_LA_InboundDoorBellRegister_T; - - -/* - Define the structure of the DAC960 LA Series Outbound Door Bell Register. -*/ - -typedef union DAC960_LA_OutboundDoorBellRegister -{ - unsigned char All; - struct { - bool AcknowledgeHardwareMailboxInterrupt:1; /* Bit 0 */ - bool AcknowledgeMemoryMailboxInterrupt:1; /* Bit 1 */ - unsigned char :6; /* Bits 2-7 */ - } Write; - struct { - bool HardwareMailboxStatusAvailable:1; /* Bit 0 */ - bool MemoryMailboxStatusAvailable:1; /* Bit 1 */ - unsigned char :6; /* Bits 2-7 */ - } Read; -} -DAC960_LA_OutboundDoorBellRegister_T; - - -/* - Define the structure of the DAC960 LA Series Interrupt Mask Register. -*/ - -typedef union DAC960_LA_InterruptMaskRegister -{ - unsigned char All; - struct { - unsigned char :2; /* Bits 0-1 */ - bool DisableInterrupts:1; /* Bit 2 */ - unsigned char :5; /* Bits 3-7 */ - } Bits; -} -DAC960_LA_InterruptMaskRegister_T; - - -/* - Define the structure of the DAC960 LA Series Error Status Register. -*/ - -typedef union DAC960_LA_ErrorStatusRegister -{ - unsigned char All; - struct { - unsigned int :2; /* Bits 0-1 */ - bool ErrorStatusPending:1; /* Bit 2 */ - unsigned int :5; /* Bits 3-7 */ - } Bits; -} -DAC960_LA_ErrorStatusRegister_T; - - -/* - Define inline functions to provide an abstraction for reading and writing the - DAC960 LA Series Controller Interface Registers. -*/ - -static inline -void DAC960_LA_HardwareMailboxNewCommand(void __iomem *ControllerBaseAddress) -{ - DAC960_LA_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.HardwareMailboxNewCommand = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LA_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_LA_AcknowledgeHardwareMailboxStatus(void __iomem *ControllerBaseAddress) -{ - DAC960_LA_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.AcknowledgeHardwareMailboxStatus = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LA_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_LA_GenerateInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_LA_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.GenerateInterrupt = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LA_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_LA_ControllerReset(void __iomem *ControllerBaseAddress) -{ - DAC960_LA_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.ControllerReset = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LA_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_LA_MemoryMailboxNewCommand(void __iomem *ControllerBaseAddress) -{ - DAC960_LA_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.MemoryMailboxNewCommand = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LA_InboundDoorBellRegisterOffset); -} - -static inline -bool DAC960_LA_HardwareMailboxFullP(void __iomem *ControllerBaseAddress) -{ - DAC960_LA_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = - readb(ControllerBaseAddress + DAC960_LA_InboundDoorBellRegisterOffset); - return !InboundDoorBellRegister.Read.HardwareMailboxEmpty; -} - -static inline -bool DAC960_LA_InitializationInProgressP(void __iomem *ControllerBaseAddress) -{ - DAC960_LA_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = - readb(ControllerBaseAddress + DAC960_LA_InboundDoorBellRegisterOffset); - return !InboundDoorBellRegister.Read.InitializationNotInProgress; -} - -static inline -void DAC960_LA_AcknowledgeHardwareMailboxInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_LA_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeHardwareMailboxInterrupt = true; - writeb(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LA_OutboundDoorBellRegisterOffset); -} - -static inline -void DAC960_LA_AcknowledgeMemoryMailboxInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_LA_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeMemoryMailboxInterrupt = true; - writeb(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LA_OutboundDoorBellRegisterOffset); -} - -static inline -void DAC960_LA_AcknowledgeInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_LA_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeHardwareMailboxInterrupt = true; - OutboundDoorBellRegister.Write.AcknowledgeMemoryMailboxInterrupt = true; - writeb(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_LA_OutboundDoorBellRegisterOffset); -} - -static inline -bool DAC960_LA_HardwareMailboxStatusAvailableP(void __iomem *ControllerBaseAddress) -{ - DAC960_LA_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = - readb(ControllerBaseAddress + DAC960_LA_OutboundDoorBellRegisterOffset); - return OutboundDoorBellRegister.Read.HardwareMailboxStatusAvailable; -} - -static inline -bool DAC960_LA_MemoryMailboxStatusAvailableP(void __iomem *ControllerBaseAddress) -{ - DAC960_LA_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = - readb(ControllerBaseAddress + DAC960_LA_OutboundDoorBellRegisterOffset); - return OutboundDoorBellRegister.Read.MemoryMailboxStatusAvailable; -} - -static inline -void DAC960_LA_EnableInterrupts(void __iomem *ControllerBaseAddress) -{ - DAC960_LA_InterruptMaskRegister_T InterruptMaskRegister; - InterruptMaskRegister.All = 0xFF; - InterruptMaskRegister.Bits.DisableInterrupts = false; - writeb(InterruptMaskRegister.All, - ControllerBaseAddress + DAC960_LA_InterruptMaskRegisterOffset); -} - -static inline -void DAC960_LA_DisableInterrupts(void __iomem *ControllerBaseAddress) -{ - DAC960_LA_InterruptMaskRegister_T InterruptMaskRegister; - InterruptMaskRegister.All = 0xFF; - InterruptMaskRegister.Bits.DisableInterrupts = true; - writeb(InterruptMaskRegister.All, - ControllerBaseAddress + DAC960_LA_InterruptMaskRegisterOffset); -} - -static inline -bool DAC960_LA_InterruptsEnabledP(void __iomem *ControllerBaseAddress) -{ - DAC960_LA_InterruptMaskRegister_T InterruptMaskRegister; - InterruptMaskRegister.All = - readb(ControllerBaseAddress + DAC960_LA_InterruptMaskRegisterOffset); - return !InterruptMaskRegister.Bits.DisableInterrupts; -} - -static inline -void DAC960_LA_WriteCommandMailbox(DAC960_V1_CommandMailbox_T - *MemoryCommandMailbox, - DAC960_V1_CommandMailbox_T - *CommandMailbox) -{ - MemoryCommandMailbox->Words[1] = CommandMailbox->Words[1]; - MemoryCommandMailbox->Words[2] = CommandMailbox->Words[2]; - MemoryCommandMailbox->Words[3] = CommandMailbox->Words[3]; - wmb(); - MemoryCommandMailbox->Words[0] = CommandMailbox->Words[0]; - mb(); -} - -static inline -void DAC960_LA_WriteHardwareMailbox(void __iomem *ControllerBaseAddress, - DAC960_V1_CommandMailbox_T *CommandMailbox) -{ - writel(CommandMailbox->Words[0], - ControllerBaseAddress + DAC960_LA_CommandOpcodeRegisterOffset); - writel(CommandMailbox->Words[1], - ControllerBaseAddress + DAC960_LA_MailboxRegister4Offset); - writel(CommandMailbox->Words[2], - ControllerBaseAddress + DAC960_LA_MailboxRegister8Offset); - writeb(CommandMailbox->Bytes[12], - ControllerBaseAddress + DAC960_LA_MailboxRegister12Offset); -} - -static inline DAC960_V1_CommandIdentifier_T -DAC960_LA_ReadStatusCommandIdentifier(void __iomem *ControllerBaseAddress) -{ - return readb(ControllerBaseAddress - + DAC960_LA_StatusCommandIdentifierRegOffset); -} - -static inline DAC960_V1_CommandStatus_T -DAC960_LA_ReadStatusRegister(void __iomem *ControllerBaseAddress) -{ - return readw(ControllerBaseAddress + DAC960_LA_StatusRegisterOffset); -} - -static inline bool -DAC960_LA_ReadErrorStatus(void __iomem *ControllerBaseAddress, - unsigned char *ErrorStatus, - unsigned char *Parameter0, - unsigned char *Parameter1) -{ - DAC960_LA_ErrorStatusRegister_T ErrorStatusRegister; - ErrorStatusRegister.All = - readb(ControllerBaseAddress + DAC960_LA_ErrorStatusRegisterOffset); - if (!ErrorStatusRegister.Bits.ErrorStatusPending) return false; - ErrorStatusRegister.Bits.ErrorStatusPending = false; - *ErrorStatus = ErrorStatusRegister.All; - *Parameter0 = - readb(ControllerBaseAddress + DAC960_LA_CommandOpcodeRegisterOffset); - *Parameter1 = - readb(ControllerBaseAddress + DAC960_LA_CommandIdentifierRegisterOffset); - writeb(0xFF, ControllerBaseAddress + DAC960_LA_ErrorStatusRegisterOffset); - return true; -} - -/* - Define the DAC960 PG Series Controller Interface Register Offsets. -*/ - -#define DAC960_PG_RegisterWindowSize 0x2000 - -typedef enum -{ - DAC960_PG_InboundDoorBellRegisterOffset = 0x0020, - DAC960_PG_OutboundDoorBellRegisterOffset = 0x002C, - DAC960_PG_InterruptMaskRegisterOffset = 0x0034, - DAC960_PG_CommandOpcodeRegisterOffset = 0x1000, - DAC960_PG_CommandIdentifierRegisterOffset = 0x1001, - DAC960_PG_MailboxRegister2Offset = 0x1002, - DAC960_PG_MailboxRegister3Offset = 0x1003, - DAC960_PG_MailboxRegister4Offset = 0x1004, - DAC960_PG_MailboxRegister5Offset = 0x1005, - DAC960_PG_MailboxRegister6Offset = 0x1006, - DAC960_PG_MailboxRegister7Offset = 0x1007, - DAC960_PG_MailboxRegister8Offset = 0x1008, - DAC960_PG_MailboxRegister9Offset = 0x1009, - DAC960_PG_MailboxRegister10Offset = 0x100A, - DAC960_PG_MailboxRegister11Offset = 0x100B, - DAC960_PG_MailboxRegister12Offset = 0x100C, - DAC960_PG_StatusCommandIdentifierRegOffset = 0x1018, - DAC960_PG_StatusRegisterOffset = 0x101A, - DAC960_PG_ErrorStatusRegisterOffset = 0x103F -} -DAC960_PG_RegisterOffsets_T; - - -/* - Define the structure of the DAC960 PG Series Inbound Door Bell Register. -*/ - -typedef union DAC960_PG_InboundDoorBellRegister -{ - unsigned int All; - struct { - bool HardwareMailboxNewCommand:1; /* Bit 0 */ - bool AcknowledgeHardwareMailboxStatus:1; /* Bit 1 */ - bool GenerateInterrupt:1; /* Bit 2 */ - bool ControllerReset:1; /* Bit 3 */ - bool MemoryMailboxNewCommand:1; /* Bit 4 */ - unsigned int :27; /* Bits 5-31 */ - } Write; - struct { - bool HardwareMailboxFull:1; /* Bit 0 */ - bool InitializationInProgress:1; /* Bit 1 */ - unsigned int :30; /* Bits 2-31 */ - } Read; -} -DAC960_PG_InboundDoorBellRegister_T; - - -/* - Define the structure of the DAC960 PG Series Outbound Door Bell Register. -*/ - -typedef union DAC960_PG_OutboundDoorBellRegister -{ - unsigned int All; - struct { - bool AcknowledgeHardwareMailboxInterrupt:1; /* Bit 0 */ - bool AcknowledgeMemoryMailboxInterrupt:1; /* Bit 1 */ - unsigned int :30; /* Bits 2-31 */ - } Write; - struct { - bool HardwareMailboxStatusAvailable:1; /* Bit 0 */ - bool MemoryMailboxStatusAvailable:1; /* Bit 1 */ - unsigned int :30; /* Bits 2-31 */ - } Read; -} -DAC960_PG_OutboundDoorBellRegister_T; - - -/* - Define the structure of the DAC960 PG Series Interrupt Mask Register. -*/ - -typedef union DAC960_PG_InterruptMaskRegister -{ - unsigned int All; - struct { - unsigned int MessageUnitInterruptMask1:2; /* Bits 0-1 */ - bool DisableInterrupts:1; /* Bit 2 */ - unsigned int MessageUnitInterruptMask2:5; /* Bits 3-7 */ - unsigned int Reserved0:24; /* Bits 8-31 */ - } Bits; -} -DAC960_PG_InterruptMaskRegister_T; - - -/* - Define the structure of the DAC960 PG Series Error Status Register. -*/ - -typedef union DAC960_PG_ErrorStatusRegister -{ - unsigned char All; - struct { - unsigned int :2; /* Bits 0-1 */ - bool ErrorStatusPending:1; /* Bit 2 */ - unsigned int :5; /* Bits 3-7 */ - } Bits; -} -DAC960_PG_ErrorStatusRegister_T; - - -/* - Define inline functions to provide an abstraction for reading and writing the - DAC960 PG Series Controller Interface Registers. -*/ - -static inline -void DAC960_PG_HardwareMailboxNewCommand(void __iomem *ControllerBaseAddress) -{ - DAC960_PG_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.HardwareMailboxNewCommand = true; - writel(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_PG_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_PG_AcknowledgeHardwareMailboxStatus(void __iomem *ControllerBaseAddress) -{ - DAC960_PG_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.AcknowledgeHardwareMailboxStatus = true; - writel(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_PG_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_PG_GenerateInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_PG_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.GenerateInterrupt = true; - writel(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_PG_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_PG_ControllerReset(void __iomem *ControllerBaseAddress) -{ - DAC960_PG_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.ControllerReset = true; - writel(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_PG_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_PG_MemoryMailboxNewCommand(void __iomem *ControllerBaseAddress) -{ - DAC960_PG_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.MemoryMailboxNewCommand = true; - writel(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_PG_InboundDoorBellRegisterOffset); -} - -static inline -bool DAC960_PG_HardwareMailboxFullP(void __iomem *ControllerBaseAddress) -{ - DAC960_PG_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = - readl(ControllerBaseAddress + DAC960_PG_InboundDoorBellRegisterOffset); - return InboundDoorBellRegister.Read.HardwareMailboxFull; -} - -static inline -bool DAC960_PG_InitializationInProgressP(void __iomem *ControllerBaseAddress) -{ - DAC960_PG_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = - readl(ControllerBaseAddress + DAC960_PG_InboundDoorBellRegisterOffset); - return InboundDoorBellRegister.Read.InitializationInProgress; -} - -static inline -void DAC960_PG_AcknowledgeHardwareMailboxInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_PG_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeHardwareMailboxInterrupt = true; - writel(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_PG_OutboundDoorBellRegisterOffset); -} - -static inline -void DAC960_PG_AcknowledgeMemoryMailboxInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_PG_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeMemoryMailboxInterrupt = true; - writel(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_PG_OutboundDoorBellRegisterOffset); -} - -static inline -void DAC960_PG_AcknowledgeInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_PG_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeHardwareMailboxInterrupt = true; - OutboundDoorBellRegister.Write.AcknowledgeMemoryMailboxInterrupt = true; - writel(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_PG_OutboundDoorBellRegisterOffset); -} - -static inline -bool DAC960_PG_HardwareMailboxStatusAvailableP(void __iomem *ControllerBaseAddress) -{ - DAC960_PG_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = - readl(ControllerBaseAddress + DAC960_PG_OutboundDoorBellRegisterOffset); - return OutboundDoorBellRegister.Read.HardwareMailboxStatusAvailable; -} - -static inline -bool DAC960_PG_MemoryMailboxStatusAvailableP(void __iomem *ControllerBaseAddress) -{ - DAC960_PG_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = - readl(ControllerBaseAddress + DAC960_PG_OutboundDoorBellRegisterOffset); - return OutboundDoorBellRegister.Read.MemoryMailboxStatusAvailable; -} - -static inline -void DAC960_PG_EnableInterrupts(void __iomem *ControllerBaseAddress) -{ - DAC960_PG_InterruptMaskRegister_T InterruptMaskRegister; - InterruptMaskRegister.All = 0; - InterruptMaskRegister.Bits.MessageUnitInterruptMask1 = 0x3; - InterruptMaskRegister.Bits.DisableInterrupts = false; - InterruptMaskRegister.Bits.MessageUnitInterruptMask2 = 0x1F; - writel(InterruptMaskRegister.All, - ControllerBaseAddress + DAC960_PG_InterruptMaskRegisterOffset); -} - -static inline -void DAC960_PG_DisableInterrupts(void __iomem *ControllerBaseAddress) -{ - DAC960_PG_InterruptMaskRegister_T InterruptMaskRegister; - InterruptMaskRegister.All = 0; - InterruptMaskRegister.Bits.MessageUnitInterruptMask1 = 0x3; - InterruptMaskRegister.Bits.DisableInterrupts = true; - InterruptMaskRegister.Bits.MessageUnitInterruptMask2 = 0x1F; - writel(InterruptMaskRegister.All, - ControllerBaseAddress + DAC960_PG_InterruptMaskRegisterOffset); -} - -static inline -bool DAC960_PG_InterruptsEnabledP(void __iomem *ControllerBaseAddress) -{ - DAC960_PG_InterruptMaskRegister_T InterruptMaskRegister; - InterruptMaskRegister.All = - readl(ControllerBaseAddress + DAC960_PG_InterruptMaskRegisterOffset); - return !InterruptMaskRegister.Bits.DisableInterrupts; -} - -static inline -void DAC960_PG_WriteCommandMailbox(DAC960_V1_CommandMailbox_T - *MemoryCommandMailbox, - DAC960_V1_CommandMailbox_T - *CommandMailbox) -{ - MemoryCommandMailbox->Words[1] = CommandMailbox->Words[1]; - MemoryCommandMailbox->Words[2] = CommandMailbox->Words[2]; - MemoryCommandMailbox->Words[3] = CommandMailbox->Words[3]; - wmb(); - MemoryCommandMailbox->Words[0] = CommandMailbox->Words[0]; - mb(); -} - -static inline -void DAC960_PG_WriteHardwareMailbox(void __iomem *ControllerBaseAddress, - DAC960_V1_CommandMailbox_T *CommandMailbox) -{ - writel(CommandMailbox->Words[0], - ControllerBaseAddress + DAC960_PG_CommandOpcodeRegisterOffset); - writel(CommandMailbox->Words[1], - ControllerBaseAddress + DAC960_PG_MailboxRegister4Offset); - writel(CommandMailbox->Words[2], - ControllerBaseAddress + DAC960_PG_MailboxRegister8Offset); - writeb(CommandMailbox->Bytes[12], - ControllerBaseAddress + DAC960_PG_MailboxRegister12Offset); -} - -static inline DAC960_V1_CommandIdentifier_T -DAC960_PG_ReadStatusCommandIdentifier(void __iomem *ControllerBaseAddress) -{ - return readb(ControllerBaseAddress - + DAC960_PG_StatusCommandIdentifierRegOffset); -} - -static inline DAC960_V1_CommandStatus_T -DAC960_PG_ReadStatusRegister(void __iomem *ControllerBaseAddress) -{ - return readw(ControllerBaseAddress + DAC960_PG_StatusRegisterOffset); -} - -static inline bool -DAC960_PG_ReadErrorStatus(void __iomem *ControllerBaseAddress, - unsigned char *ErrorStatus, - unsigned char *Parameter0, - unsigned char *Parameter1) -{ - DAC960_PG_ErrorStatusRegister_T ErrorStatusRegister; - ErrorStatusRegister.All = - readb(ControllerBaseAddress + DAC960_PG_ErrorStatusRegisterOffset); - if (!ErrorStatusRegister.Bits.ErrorStatusPending) return false; - ErrorStatusRegister.Bits.ErrorStatusPending = false; - *ErrorStatus = ErrorStatusRegister.All; - *Parameter0 = - readb(ControllerBaseAddress + DAC960_PG_CommandOpcodeRegisterOffset); - *Parameter1 = - readb(ControllerBaseAddress + DAC960_PG_CommandIdentifierRegisterOffset); - writeb(0, ControllerBaseAddress + DAC960_PG_ErrorStatusRegisterOffset); - return true; -} - -/* - Define the DAC960 PD Series Controller Interface Register Offsets. -*/ - -#define DAC960_PD_RegisterWindowSize 0x80 - -typedef enum -{ - DAC960_PD_CommandOpcodeRegisterOffset = 0x00, - DAC960_PD_CommandIdentifierRegisterOffset = 0x01, - DAC960_PD_MailboxRegister2Offset = 0x02, - DAC960_PD_MailboxRegister3Offset = 0x03, - DAC960_PD_MailboxRegister4Offset = 0x04, - DAC960_PD_MailboxRegister5Offset = 0x05, - DAC960_PD_MailboxRegister6Offset = 0x06, - DAC960_PD_MailboxRegister7Offset = 0x07, - DAC960_PD_MailboxRegister8Offset = 0x08, - DAC960_PD_MailboxRegister9Offset = 0x09, - DAC960_PD_MailboxRegister10Offset = 0x0A, - DAC960_PD_MailboxRegister11Offset = 0x0B, - DAC960_PD_MailboxRegister12Offset = 0x0C, - DAC960_PD_StatusCommandIdentifierRegOffset = 0x0D, - DAC960_PD_StatusRegisterOffset = 0x0E, - DAC960_PD_ErrorStatusRegisterOffset = 0x3F, - DAC960_PD_InboundDoorBellRegisterOffset = 0x40, - DAC960_PD_OutboundDoorBellRegisterOffset = 0x41, - DAC960_PD_InterruptEnableRegisterOffset = 0x43 -} -DAC960_PD_RegisterOffsets_T; - - -/* - Define the structure of the DAC960 PD Series Inbound Door Bell Register. -*/ - -typedef union DAC960_PD_InboundDoorBellRegister -{ - unsigned char All; - struct { - bool NewCommand:1; /* Bit 0 */ - bool AcknowledgeStatus:1; /* Bit 1 */ - bool GenerateInterrupt:1; /* Bit 2 */ - bool ControllerReset:1; /* Bit 3 */ - unsigned char :4; /* Bits 4-7 */ - } Write; - struct { - bool MailboxFull:1; /* Bit 0 */ - bool InitializationInProgress:1; /* Bit 1 */ - unsigned char :6; /* Bits 2-7 */ - } Read; -} -DAC960_PD_InboundDoorBellRegister_T; - - -/* - Define the structure of the DAC960 PD Series Outbound Door Bell Register. -*/ - -typedef union DAC960_PD_OutboundDoorBellRegister -{ - unsigned char All; - struct { - bool AcknowledgeInterrupt:1; /* Bit 0 */ - unsigned char :7; /* Bits 1-7 */ - } Write; - struct { - bool StatusAvailable:1; /* Bit 0 */ - unsigned char :7; /* Bits 1-7 */ - } Read; -} -DAC960_PD_OutboundDoorBellRegister_T; - - -/* - Define the structure of the DAC960 PD Series Interrupt Enable Register. -*/ - -typedef union DAC960_PD_InterruptEnableRegister -{ - unsigned char All; - struct { - bool EnableInterrupts:1; /* Bit 0 */ - unsigned char :7; /* Bits 1-7 */ - } Bits; -} -DAC960_PD_InterruptEnableRegister_T; - - -/* - Define the structure of the DAC960 PD Series Error Status Register. -*/ - -typedef union DAC960_PD_ErrorStatusRegister -{ - unsigned char All; - struct { - unsigned int :2; /* Bits 0-1 */ - bool ErrorStatusPending:1; /* Bit 2 */ - unsigned int :5; /* Bits 3-7 */ - } Bits; -} -DAC960_PD_ErrorStatusRegister_T; - - -/* - Define inline functions to provide an abstraction for reading and writing the - DAC960 PD Series Controller Interface Registers. -*/ - -static inline -void DAC960_PD_NewCommand(void __iomem *ControllerBaseAddress) -{ - DAC960_PD_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.NewCommand = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_PD_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_PD_AcknowledgeStatus(void __iomem *ControllerBaseAddress) -{ - DAC960_PD_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.AcknowledgeStatus = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_PD_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_PD_GenerateInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_PD_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.GenerateInterrupt = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_PD_InboundDoorBellRegisterOffset); -} - -static inline -void DAC960_PD_ControllerReset(void __iomem *ControllerBaseAddress) -{ - DAC960_PD_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = 0; - InboundDoorBellRegister.Write.ControllerReset = true; - writeb(InboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_PD_InboundDoorBellRegisterOffset); -} - -static inline -bool DAC960_PD_MailboxFullP(void __iomem *ControllerBaseAddress) -{ - DAC960_PD_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = - readb(ControllerBaseAddress + DAC960_PD_InboundDoorBellRegisterOffset); - return InboundDoorBellRegister.Read.MailboxFull; -} - -static inline -bool DAC960_PD_InitializationInProgressP(void __iomem *ControllerBaseAddress) -{ - DAC960_PD_InboundDoorBellRegister_T InboundDoorBellRegister; - InboundDoorBellRegister.All = - readb(ControllerBaseAddress + DAC960_PD_InboundDoorBellRegisterOffset); - return InboundDoorBellRegister.Read.InitializationInProgress; -} - -static inline -void DAC960_PD_AcknowledgeInterrupt(void __iomem *ControllerBaseAddress) -{ - DAC960_PD_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = 0; - OutboundDoorBellRegister.Write.AcknowledgeInterrupt = true; - writeb(OutboundDoorBellRegister.All, - ControllerBaseAddress + DAC960_PD_OutboundDoorBellRegisterOffset); -} - -static inline -bool DAC960_PD_StatusAvailableP(void __iomem *ControllerBaseAddress) -{ - DAC960_PD_OutboundDoorBellRegister_T OutboundDoorBellRegister; - OutboundDoorBellRegister.All = - readb(ControllerBaseAddress + DAC960_PD_OutboundDoorBellRegisterOffset); - return OutboundDoorBellRegister.Read.StatusAvailable; -} - -static inline -void DAC960_PD_EnableInterrupts(void __iomem *ControllerBaseAddress) -{ - DAC960_PD_InterruptEnableRegister_T InterruptEnableRegister; - InterruptEnableRegister.All = 0; - InterruptEnableRegister.Bits.EnableInterrupts = true; - writeb(InterruptEnableRegister.All, - ControllerBaseAddress + DAC960_PD_InterruptEnableRegisterOffset); -} - -static inline -void DAC960_PD_DisableInterrupts(void __iomem *ControllerBaseAddress) -{ - DAC960_PD_InterruptEnableRegister_T InterruptEnableRegister; - InterruptEnableRegister.All = 0; - InterruptEnableRegister.Bits.EnableInterrupts = false; - writeb(InterruptEnableRegister.All, - ControllerBaseAddress + DAC960_PD_InterruptEnableRegisterOffset); -} - -static inline -bool DAC960_PD_InterruptsEnabledP(void __iomem *ControllerBaseAddress) -{ - DAC960_PD_InterruptEnableRegister_T InterruptEnableRegister; - InterruptEnableRegister.All = - readb(ControllerBaseAddress + DAC960_PD_InterruptEnableRegisterOffset); - return InterruptEnableRegister.Bits.EnableInterrupts; -} - -static inline -void DAC960_PD_WriteCommandMailbox(void __iomem *ControllerBaseAddress, - DAC960_V1_CommandMailbox_T *CommandMailbox) -{ - writel(CommandMailbox->Words[0], - ControllerBaseAddress + DAC960_PD_CommandOpcodeRegisterOffset); - writel(CommandMailbox->Words[1], - ControllerBaseAddress + DAC960_PD_MailboxRegister4Offset); - writel(CommandMailbox->Words[2], - ControllerBaseAddress + DAC960_PD_MailboxRegister8Offset); - writeb(CommandMailbox->Bytes[12], - ControllerBaseAddress + DAC960_PD_MailboxRegister12Offset); -} - -static inline DAC960_V1_CommandIdentifier_T -DAC960_PD_ReadStatusCommandIdentifier(void __iomem *ControllerBaseAddress) -{ - return readb(ControllerBaseAddress - + DAC960_PD_StatusCommandIdentifierRegOffset); -} - -static inline DAC960_V1_CommandStatus_T -DAC960_PD_ReadStatusRegister(void __iomem *ControllerBaseAddress) -{ - return readw(ControllerBaseAddress + DAC960_PD_StatusRegisterOffset); -} - -static inline bool -DAC960_PD_ReadErrorStatus(void __iomem *ControllerBaseAddress, - unsigned char *ErrorStatus, - unsigned char *Parameter0, - unsigned char *Parameter1) -{ - DAC960_PD_ErrorStatusRegister_T ErrorStatusRegister; - ErrorStatusRegister.All = - readb(ControllerBaseAddress + DAC960_PD_ErrorStatusRegisterOffset); - if (!ErrorStatusRegister.Bits.ErrorStatusPending) return false; - ErrorStatusRegister.Bits.ErrorStatusPending = false; - *ErrorStatus = ErrorStatusRegister.All; - *Parameter0 = - readb(ControllerBaseAddress + DAC960_PD_CommandOpcodeRegisterOffset); - *Parameter1 = - readb(ControllerBaseAddress + DAC960_PD_CommandIdentifierRegisterOffset); - writeb(0, ControllerBaseAddress + DAC960_PD_ErrorStatusRegisterOffset); - return true; -} - -static inline void DAC960_P_To_PD_TranslateEnquiry(void *Enquiry) -{ - memcpy(Enquiry + 132, Enquiry + 36, 64); - memset(Enquiry + 36, 0, 96); -} - -static inline void DAC960_P_To_PD_TranslateDeviceState(void *DeviceState) -{ - memcpy(DeviceState + 2, DeviceState + 3, 1); - memmove(DeviceState + 4, DeviceState + 5, 2); - memmove(DeviceState + 6, DeviceState + 8, 4); -} - -static inline -void DAC960_PD_To_P_TranslateReadWriteCommand(DAC960_V1_CommandMailbox_T - *CommandMailbox) -{ - int LogicalDriveNumber = CommandMailbox->Type5.LD.LogicalDriveNumber; - CommandMailbox->Bytes[3] &= 0x7; - CommandMailbox->Bytes[3] |= CommandMailbox->Bytes[7] << 6; - CommandMailbox->Bytes[7] = LogicalDriveNumber; -} - -static inline -void DAC960_P_To_PD_TranslateReadWriteCommand(DAC960_V1_CommandMailbox_T - *CommandMailbox) -{ - int LogicalDriveNumber = CommandMailbox->Bytes[7]; - CommandMailbox->Bytes[7] = CommandMailbox->Bytes[3] >> 6; - CommandMailbox->Bytes[3] &= 0x7; - CommandMailbox->Bytes[3] |= LogicalDriveNumber << 3; -} - - -/* - Define prototypes for the forward referenced DAC960 Driver Internal Functions. -*/ - -static void DAC960_FinalizeController(DAC960_Controller_T *); -static void DAC960_V1_QueueReadWriteCommand(DAC960_Command_T *); -static void DAC960_V2_QueueReadWriteCommand(DAC960_Command_T *); -static void DAC960_RequestFunction(struct request_queue *); -static irqreturn_t DAC960_BA_InterruptHandler(int, void *); -static irqreturn_t DAC960_LP_InterruptHandler(int, void *); -static irqreturn_t DAC960_LA_InterruptHandler(int, void *); -static irqreturn_t DAC960_PG_InterruptHandler(int, void *); -static irqreturn_t DAC960_PD_InterruptHandler(int, void *); -static irqreturn_t DAC960_P_InterruptHandler(int, void *); -static void DAC960_V1_QueueMonitoringCommand(DAC960_Command_T *); -static void DAC960_V2_QueueMonitoringCommand(DAC960_Command_T *); -static void DAC960_MonitoringTimerFunction(struct timer_list *); -static void DAC960_Message(DAC960_MessageLevel_T, unsigned char *, - DAC960_Controller_T *, ...); -static void DAC960_CreateProcEntries(DAC960_Controller_T *); -static void DAC960_DestroyProcEntries(DAC960_Controller_T *); - -#endif /* DAC960_DriverVersion */ diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig index d4913516823f..20bb4bfa4be6 100644 --- a/drivers/block/Kconfig +++ b/drivers/block/Kconfig @@ -121,18 +121,6 @@ source "drivers/block/mtip32xx/Kconfig" source "drivers/block/zram/Kconfig" -config BLK_DEV_DAC960 - tristate "Mylex DAC960/DAC1100 PCI RAID Controller support" - depends on PCI - help - This driver adds support for the Mylex DAC960, AcceleRAID, and - eXtremeRAID PCI RAID controllers. See the file - <file:Documentation/blockdev/README.DAC960> for further information - about this driver. - - To compile this driver as a module, choose M here: the - module will be called DAC960. - config BLK_DEV_UMEM tristate "Micro Memory MM5415 Battery Backed RAM support" depends on PCI @@ -461,7 +449,6 @@ config BLK_DEV_RBD select LIBCRC32C select CRYPTO_AES select CRYPTO - default n help Say Y here if you want include the Rados block device, which stripes a block device over objects stored in the Ceph distributed object diff --git a/drivers/block/Makefile b/drivers/block/Makefile index 8566b188368b..a53cc1e3a2d3 100644 --- a/drivers/block/Makefile +++ b/drivers/block/Makefile @@ -16,7 +16,6 @@ obj-$(CONFIG_ATARI_FLOPPY) += ataflop.o obj-$(CONFIG_AMIGA_Z2RAM) += z2ram.o obj-$(CONFIG_BLK_DEV_RAM) += brd.o obj-$(CONFIG_BLK_DEV_LOOP) += loop.o -obj-$(CONFIG_BLK_DEV_DAC960) += DAC960.o obj-$(CONFIG_XILINX_SYSACE) += xsysace.o obj-$(CONFIG_CDROM_PKTCDVD) += pktcdvd.o obj-$(CONFIG_SUNVDC) += sunvdc.o diff --git a/drivers/block/amiflop.c b/drivers/block/amiflop.c index 3aaf6af3ec23..bf996bd44cfc 100644 --- a/drivers/block/amiflop.c +++ b/drivers/block/amiflop.c @@ -61,10 +61,8 @@ #include <linux/delay.h> #include <linux/init.h> #include <linux/mutex.h> -#include <linux/amifdreg.h> -#include <linux/amifd.h> #include <linux/fs.h> -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/elevator.h> #include <linux/interrupt.h> #include <linux/platform_device.h> @@ -87,6 +85,126 @@ */ /* + * CIAAPRA bits (read only) + */ + +#define DSKRDY (0x1<<5) /* disk ready when low */ +#define DSKTRACK0 (0x1<<4) /* head at track zero when low */ +#define DSKPROT (0x1<<3) /* disk protected when low */ +#define DSKCHANGE (0x1<<2) /* low when disk removed */ + +/* + * CIAAPRB bits (read/write) + */ + +#define DSKMOTOR (0x1<<7) /* motor on when low */ +#define DSKSEL3 (0x1<<6) /* select drive 3 when low */ +#define DSKSEL2 (0x1<<5) /* select drive 2 when low */ +#define DSKSEL1 (0x1<<4) /* select drive 1 when low */ +#define DSKSEL0 (0x1<<3) /* select drive 0 when low */ +#define DSKSIDE (0x1<<2) /* side selection: 0 = upper, 1 = lower */ +#define DSKDIREC (0x1<<1) /* step direction: 0=in, 1=out (to trk 0) */ +#define DSKSTEP (0x1) /* pulse low to step head 1 track */ + +/* + * DSKBYTR bits (read only) + */ + +#define DSKBYT (1<<15) /* register contains valid byte when set */ +#define DMAON (1<<14) /* disk DMA enabled */ +#define DISKWRITE (1<<13) /* disk write bit in DSKLEN enabled */ +#define WORDEQUAL (1<<12) /* DSKSYNC register match when true */ +/* bits 7-0 are data */ + +/* + * ADKCON/ADKCONR bits + */ + +#ifndef SETCLR +#define ADK_SETCLR (1<<15) /* control bit */ +#endif +#define ADK_PRECOMP1 (1<<14) /* precompensation selection */ +#define ADK_PRECOMP0 (1<<13) /* 00=none, 01=140ns, 10=280ns, 11=500ns */ +#define ADK_MFMPREC (1<<12) /* 0=GCR precomp., 1=MFM precomp. */ +#define ADK_WORDSYNC (1<<10) /* enable DSKSYNC auto DMA */ +#define ADK_MSBSYNC (1<<9) /* when 1, enable sync on MSbit (for GCR) */ +#define ADK_FAST (1<<8) /* bit cell: 0=2us (GCR), 1=1us (MFM) */ + +/* + * DSKLEN bits + */ + +#define DSKLEN_DMAEN (1<<15) +#define DSKLEN_WRITE (1<<14) + +/* + * INTENA/INTREQ bits + */ + +#define DSKINDEX (0x1<<4) /* DSKINDEX bit */ + +/* + * Misc + */ + +#define MFM_SYNC 0x4489 /* standard MFM sync value */ + +/* Values for FD_COMMAND */ +#define FD_RECALIBRATE 0x07 /* move to track 0 */ +#define FD_SEEK 0x0F /* seek track */ +#define FD_READ 0xE6 /* read with MT, MFM, SKip deleted */ +#define FD_WRITE 0xC5 /* write with MT, MFM */ +#define FD_SENSEI 0x08 /* Sense Interrupt Status */ +#define FD_SPECIFY 0x03 /* specify HUT etc */ +#define FD_FORMAT 0x4D /* format one track */ +#define FD_VERSION 0x10 /* get version code */ +#define FD_CONFIGURE 0x13 /* configure FIFO operation */ +#define FD_PERPENDICULAR 0x12 /* perpendicular r/w mode */ + +#define FD_MAX_UNITS 4 /* Max. Number of drives */ +#define FLOPPY_MAX_SECTORS 22 /* Max. Number of sectors per track */ + +struct fd_data_type { + char *name; /* description of data type */ + int sects; /* sectors per track */ + int (*read_fkt)(int); /* read whole track */ + void (*write_fkt)(int); /* write whole track */ +}; + +struct fd_drive_type { + unsigned long code; /* code returned from drive */ + char *name; /* description of drive */ + unsigned int tracks; /* number of tracks */ + unsigned int heads; /* number of heads */ + unsigned int read_size; /* raw read size for one track */ + unsigned int write_size; /* raw write size for one track */ + unsigned int sect_mult; /* sectors and gap multiplier (HD = 2) */ + unsigned int precomp1; /* start track for precomp 1 */ + unsigned int precomp2; /* start track for precomp 2 */ + unsigned int step_delay; /* time (in ms) for delay after step */ + unsigned int settle_time; /* time to settle after dir change */ + unsigned int side_time; /* time needed to change sides */ +}; + +struct amiga_floppy_struct { + struct fd_drive_type *type; /* type of floppy for this unit */ + struct fd_data_type *dtype; /* type of floppy for this unit */ + int track; /* current track (-1 == unknown) */ + unsigned char *trackbuf; /* current track (kmaloc()'d */ + + int blocks; /* total # blocks on disk */ + + int changed; /* true when not known */ + int disk; /* disk in drive (-1 == unknown) */ + int motor; /* true when motor is at speed */ + int busy; /* true when drive is active */ + int dirty; /* true when trackbuf is not on disk */ + int status; /* current error code for unit */ + struct gendisk *gendisk; + struct blk_mq_tag_set tag_set; +}; + +/* * Error codes */ #define FD_OK 0 /* operation succeeded */ @@ -164,7 +282,6 @@ static volatile int selected = -1; /* currently selected drive */ static int writepending; static int writefromint; static char *raw_buf; -static int fdc_queue; static DEFINE_SPINLOCK(amiflop_lock); @@ -1337,76 +1454,20 @@ static int get_track(int drive, int track) return -1; } -/* - * Round-robin between our available drives, doing one request from each - */ -static struct request *set_next_request(void) -{ - struct request_queue *q; - int cnt = FD_MAX_UNITS; - struct request *rq = NULL; - - /* Find next queue we can dispatch from */ - fdc_queue = fdc_queue + 1; - if (fdc_queue == FD_MAX_UNITS) - fdc_queue = 0; - - for(cnt = FD_MAX_UNITS; cnt > 0; cnt--) { - - if (unit[fdc_queue].type->code == FD_NODRIVE) { - if (++fdc_queue == FD_MAX_UNITS) - fdc_queue = 0; - continue; - } - - q = unit[fdc_queue].gendisk->queue; - if (q) { - rq = blk_fetch_request(q); - if (rq) - break; - } - - if (++fdc_queue == FD_MAX_UNITS) - fdc_queue = 0; - } - - return rq; -} - -static void redo_fd_request(void) +static blk_status_t amiflop_rw_cur_segment(struct amiga_floppy_struct *floppy, + struct request *rq) { - struct request *rq; + int drive = floppy - unit; unsigned int cnt, block, track, sector; - int drive; - struct amiga_floppy_struct *floppy; char *data; - unsigned long flags; - blk_status_t err; - -next_req: - rq = set_next_request(); - if (!rq) { - /* Nothing left to do */ - return; - } - - floppy = rq->rq_disk->private_data; - drive = floppy - unit; -next_segment: - /* Here someone could investigate to be more efficient */ - for (cnt = 0, err = BLK_STS_OK; cnt < blk_rq_cur_sectors(rq); cnt++) { + for (cnt = 0; cnt < blk_rq_cur_sectors(rq); cnt++) { #ifdef DEBUG printk("fd: sector %ld + %d requested for %s\n", blk_rq_pos(rq), cnt, (rq_data_dir(rq) == READ) ? "read" : "write"); #endif block = blk_rq_pos(rq) + cnt; - if ((int)block > floppy->blocks) { - err = BLK_STS_IOERR; - break; - } - track = block / (floppy->dtype->sects * floppy->type->sect_mult); sector = block % (floppy->dtype->sects * floppy->type->sect_mult); data = bio_data(rq->bio) + 512 * cnt; @@ -1415,10 +1476,8 @@ next_segment: "0x%08lx\n", track, sector, data); #endif - if (get_track(drive, track) == -1) { - err = BLK_STS_IOERR; - break; - } + if (get_track(drive, track) == -1) + return BLK_STS_IOERR; if (rq_data_dir(rq) == READ) { memcpy(data, floppy->trackbuf + sector * 512, 512); @@ -1426,31 +1485,40 @@ next_segment: memcpy(floppy->trackbuf + sector * 512, data, 512); /* keep the drive spinning while writes are scheduled */ - if (!fd_motor_on(drive)) { - err = BLK_STS_IOERR; - break; - } + if (!fd_motor_on(drive)) + return BLK_STS_IOERR; /* * setup a callback to write the track buffer * after a short (1 tick) delay. */ - local_irq_save(flags); - floppy->dirty = 1; /* reset the timer */ mod_timer (flush_track_timer + drive, jiffies + 1); - local_irq_restore(flags); } } - if (__blk_end_request_cur(rq, err)) - goto next_segment; - goto next_req; + return BLK_STS_OK; } -static void do_fd_request(struct request_queue * q) +static blk_status_t amiflop_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { - redo_fd_request(); + struct request *rq = bd->rq; + struct amiga_floppy_struct *floppy = rq->rq_disk->private_data; + blk_status_t err; + + if (!spin_trylock_irq(&amiflop_lock)) + return BLK_STS_DEV_RESOURCE; + + blk_mq_start_request(rq); + + do { + err = amiflop_rw_cur_segment(floppy, rq); + } while (blk_update_request(rq, err, blk_rq_cur_bytes(rq))); + blk_mq_end_request(rq, err); + + spin_unlock_irq(&amiflop_lock); + return BLK_STS_OK; } static int fd_getgeo(struct block_device *bdev, struct hd_geometry *geo) @@ -1701,11 +1769,47 @@ static const struct block_device_operations floppy_fops = { .check_events = amiga_check_events, }; +static const struct blk_mq_ops amiflop_mq_ops = { + .queue_rq = amiflop_queue_rq, +}; + +static struct gendisk *fd_alloc_disk(int drive) +{ + struct gendisk *disk; + + disk = alloc_disk(1); + if (!disk) + goto out; + + disk->queue = blk_mq_init_sq_queue(&unit[drive].tag_set, &amiflop_mq_ops, + 2, BLK_MQ_F_SHOULD_MERGE); + if (IS_ERR(disk->queue)) { + disk->queue = NULL; + goto out_put_disk; + } + + unit[drive].trackbuf = kmalloc(FLOPPY_MAX_SECTORS * 512, GFP_KERNEL); + if (!unit[drive].trackbuf) + goto out_cleanup_queue; + + return disk; + +out_cleanup_queue: + blk_cleanup_queue(disk->queue); + disk->queue = NULL; + blk_mq_free_tag_set(&unit[drive].tag_set); +out_put_disk: + put_disk(disk); +out: + unit[drive].type->code = FD_NODRIVE; + return NULL; +} + static int __init fd_probe_drives(void) { int drive,drives,nomem; - printk(KERN_INFO "FD: probing units\nfound "); + pr_info("FD: probing units\nfound"); drives=0; nomem=0; for(drive=0;drive<FD_MAX_UNITS;drive++) { @@ -1713,27 +1817,17 @@ static int __init fd_probe_drives(void) fd_probe(drive); if (unit[drive].type->code == FD_NODRIVE) continue; - disk = alloc_disk(1); + + disk = fd_alloc_disk(drive); if (!disk) { - unit[drive].type->code = FD_NODRIVE; + pr_cont(" no mem for fd%d", drive); + nomem = 1; continue; } unit[drive].gendisk = disk; - - disk->queue = blk_init_queue(do_fd_request, &amiflop_lock); - if (!disk->queue) { - unit[drive].type->code = FD_NODRIVE; - continue; - } - drives++; - if ((unit[drive].trackbuf = kmalloc(FLOPPY_MAX_SECTORS * 512, GFP_KERNEL)) == NULL) { - printk("no mem for "); - unit[drive].type = &drive_types[num_dr_types - 1]; /* FD_NODRIVE */ - drives--; - nomem = 1; - } - printk("fd%d ",drive); + + pr_cont(" fd%d",drive); disk->major = FLOPPY_MAJOR; disk->first_minor = drive; disk->fops = &floppy_fops; @@ -1744,11 +1838,11 @@ static int __init fd_probe_drives(void) } if ((drives > 0) || (nomem == 0)) { if (drives == 0) - printk("no drives"); - printk("\n"); + pr_cont(" no drives"); + pr_cont("\n"); return drives; } - printk("\n"); + pr_cont("\n"); return -ENOMEM; } @@ -1831,30 +1925,6 @@ out_blkdev: return ret; } -#if 0 /* not safe to unload */ -static int __exit amiga_floppy_remove(struct platform_device *pdev) -{ - int i; - - for( i = 0; i < FD_MAX_UNITS; i++) { - if (unit[i].type->code != FD_NODRIVE) { - struct request_queue *q = unit[i].gendisk->queue; - del_gendisk(unit[i].gendisk); - put_disk(unit[i].gendisk); - kfree(unit[i].trackbuf); - if (q) - blk_cleanup_queue(q); - } - } - blk_unregister_region(MKDEV(FLOPPY_MAJOR, 0), 256); - free_irq(IRQ_AMIGA_CIAA_TB, NULL); - free_irq(IRQ_AMIGA_DSKBLK, NULL); - custom.dmacon = DMAF_DISK; /* disable DMA */ - amiga_chip_free(raw_buf); - unregister_blkdev(FLOPPY_MAJOR, "fd"); -} -#endif - static struct platform_driver amiga_floppy_driver = { .driver = { .name = "amiga-floppy", diff --git a/drivers/block/aoe/aoe.h b/drivers/block/aoe/aoe.h index c0ebda1283cc..7ca76ed2e71a 100644 --- a/drivers/block/aoe/aoe.h +++ b/drivers/block/aoe/aoe.h @@ -1,4 +1,6 @@ /* Copyright (c) 2013 Coraid, Inc. See COPYING for GPL terms. */ +#include <linux/blk-mq.h> + #define VERSION "85" #define AOE_MAJOR 152 #define DEVICE_NAME "aoe" @@ -164,6 +166,8 @@ struct aoedev { struct gendisk *gd; struct dentry *debugfs; struct request_queue *blkq; + struct list_head rq_list; + struct blk_mq_tag_set tag_set; struct hd_geometry geo; sector_t ssize; struct timer_list timer; @@ -201,7 +205,6 @@ int aoeblk_init(void); void aoeblk_exit(void); void aoeblk_gdalloc(void *); void aoedisk_rm_debugfs(struct aoedev *d); -void aoedisk_rm_sysfs(struct aoedev *d); int aoechr_init(void); void aoechr_exit(void); diff --git a/drivers/block/aoe/aoeblk.c b/drivers/block/aoe/aoeblk.c index 429ebb84b592..ed26b7287256 100644 --- a/drivers/block/aoe/aoeblk.c +++ b/drivers/block/aoe/aoeblk.c @@ -6,7 +6,7 @@ #include <linux/kernel.h> #include <linux/hdreg.h> -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/backing-dev.h> #include <linux/fs.h> #include <linux/ioctl.h> @@ -177,10 +177,15 @@ static struct attribute *aoe_attrs[] = { NULL, }; -static const struct attribute_group attr_group = { +static const struct attribute_group aoe_attr_group = { .attrs = aoe_attrs, }; +static const struct attribute_group *aoe_attr_groups[] = { + &aoe_attr_group, + NULL, +}; + static const struct file_operations aoe_debugfs_fops = { .open = aoe_debugfs_open, .read = seq_read, @@ -220,17 +225,6 @@ aoedisk_rm_debugfs(struct aoedev *d) } static int -aoedisk_add_sysfs(struct aoedev *d) -{ - return sysfs_create_group(&disk_to_dev(d->gd)->kobj, &attr_group); -} -void -aoedisk_rm_sysfs(struct aoedev *d) -{ - sysfs_remove_group(&disk_to_dev(d->gd)->kobj, &attr_group); -} - -static int aoeblk_open(struct block_device *bdev, fmode_t mode) { struct aoedev *d = bdev->bd_disk->private_data; @@ -274,23 +268,25 @@ aoeblk_release(struct gendisk *disk, fmode_t mode) spin_unlock_irqrestore(&d->lock, flags); } -static void -aoeblk_request(struct request_queue *q) +static blk_status_t aoeblk_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { - struct aoedev *d; - struct request *rq; + struct aoedev *d = hctx->queue->queuedata; + + spin_lock_irq(&d->lock); - d = q->queuedata; if ((d->flags & DEVFL_UP) == 0) { pr_info_ratelimited("aoe: device %ld.%d is not up\n", d->aoemajor, d->aoeminor); - while ((rq = blk_peek_request(q))) { - blk_start_request(rq); - aoe_end_request(d, rq, 1); - } - return; + spin_unlock_irq(&d->lock); + blk_mq_start_request(bd->rq); + return BLK_STS_IOERR; } + + list_add_tail(&bd->rq->queuelist, &d->rq_list); aoecmd_work(d); + spin_unlock_irq(&d->lock); + return BLK_STS_OK; } static int @@ -345,6 +341,10 @@ static const struct block_device_operations aoe_bdops = { .owner = THIS_MODULE, }; +static const struct blk_mq_ops aoeblk_mq_ops = { + .queue_rq = aoeblk_queue_rq, +}; + /* alloc_disk and add_disk can sleep */ void aoeblk_gdalloc(void *vp) @@ -353,9 +353,11 @@ aoeblk_gdalloc(void *vp) struct gendisk *gd; mempool_t *mp; struct request_queue *q; + struct blk_mq_tag_set *set; enum { KB = 1024, MB = KB * KB, READ_AHEAD = 2 * MB, }; ulong flags; int late = 0; + int err; spin_lock_irqsave(&d->lock, flags); if (d->flags & DEVFL_GDALLOC @@ -382,10 +384,25 @@ aoeblk_gdalloc(void *vp) d->aoemajor, d->aoeminor); goto err_disk; } - q = blk_init_queue(aoeblk_request, &d->lock); - if (q == NULL) { + + set = &d->tag_set; + set->ops = &aoeblk_mq_ops; + set->nr_hw_queues = 1; + set->queue_depth = 128; + set->numa_node = NUMA_NO_NODE; + set->flags = BLK_MQ_F_SHOULD_MERGE; + err = blk_mq_alloc_tag_set(set); + if (err) { + pr_err("aoe: cannot allocate tag set for %ld.%d\n", + d->aoemajor, d->aoeminor); + goto err_mempool; + } + + q = blk_mq_init_queue(set); + if (IS_ERR(q)) { pr_err("aoe: cannot allocate block queue for %ld.%d\n", d->aoemajor, d->aoeminor); + blk_mq_free_tag_set(set); goto err_mempool; } @@ -417,8 +434,7 @@ aoeblk_gdalloc(void *vp) spin_unlock_irqrestore(&d->lock, flags); - add_disk(gd); - aoedisk_add_sysfs(d); + device_add_disk(NULL, gd, aoe_attr_groups); aoedisk_add_debugfs(d); spin_lock_irqsave(&d->lock, flags); diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c index 136dc507d020..bb2fba651bd2 100644 --- a/drivers/block/aoe/aoecmd.c +++ b/drivers/block/aoe/aoecmd.c @@ -7,7 +7,7 @@ #include <linux/ata.h> #include <linux/slab.h> #include <linux/hdreg.h> -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/skbuff.h> #include <linux/netdevice.h> #include <linux/genhd.h> @@ -813,7 +813,7 @@ rexmit_timer(struct timer_list *timer) out: if ((d->flags & DEVFL_KICKME) && d->blkq) { d->flags &= ~DEVFL_KICKME; - d->blkq->request_fn(d->blkq); + blk_mq_run_hw_queues(d->blkq, true); } d->timer.expires = jiffies + TIMERTICK; @@ -857,10 +857,12 @@ nextbuf(struct aoedev *d) return d->ip.buf; rq = d->ip.rq; if (rq == NULL) { - rq = blk_peek_request(q); + rq = list_first_entry_or_null(&d->rq_list, struct request, + queuelist); if (rq == NULL) return NULL; - blk_start_request(rq); + list_del_init(&rq->queuelist); + blk_mq_start_request(rq); d->ip.rq = rq; d->ip.nxbio = rq->bio; rq->special = (void *) rqbiocnt(rq); @@ -1045,6 +1047,7 @@ aoe_end_request(struct aoedev *d, struct request *rq, int fastfail) struct bio *bio; int bok; struct request_queue *q; + blk_status_t err = BLK_STS_OK; q = d->blkq; if (rq == d->ip.rq) @@ -1052,11 +1055,15 @@ aoe_end_request(struct aoedev *d, struct request *rq, int fastfail) do { bio = rq->bio; bok = !fastfail && !bio->bi_status; - } while (__blk_end_request(rq, bok ? BLK_STS_OK : BLK_STS_IOERR, bio->bi_iter.bi_size)); + if (!bok) + err = BLK_STS_IOERR; + } while (blk_update_request(rq, bok ? BLK_STS_OK : BLK_STS_IOERR, bio->bi_iter.bi_size)); + + __blk_mq_end_request(rq, err); /* cf. http://lkml.org/lkml/2006/10/31/28 */ if (!fastfail) - __blk_run_queue(q); + blk_mq_run_hw_queues(q, true); } static void diff --git a/drivers/block/aoe/aoedev.c b/drivers/block/aoe/aoedev.c index 41060e9cedf2..9063f8efbd3b 100644 --- a/drivers/block/aoe/aoedev.c +++ b/drivers/block/aoe/aoedev.c @@ -5,7 +5,7 @@ */ #include <linux/hdreg.h> -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/netdevice.h> #include <linux/delay.h> #include <linux/slab.h> @@ -197,7 +197,6 @@ aoedev_downdev(struct aoedev *d) { struct aoetgt *t, **tt, **te; struct list_head *head, *pos, *nx; - struct request *rq; int i; d->flags &= ~DEVFL_UP; @@ -225,10 +224,11 @@ aoedev_downdev(struct aoedev *d) /* fast fail all pending I/O */ if (d->blkq) { - while ((rq = blk_peek_request(d->blkq))) { - blk_start_request(rq); - aoe_end_request(d, rq, 1); - } + /* UP is cleared, freeze+quiesce to insure all are errored */ + blk_mq_freeze_queue(d->blkq); + blk_mq_quiesce_queue(d->blkq); + blk_mq_unquiesce_queue(d->blkq); + blk_mq_unfreeze_queue(d->blkq); } if (d->gd) @@ -275,9 +275,9 @@ freedev(struct aoedev *d) del_timer_sync(&d->timer); if (d->gd) { aoedisk_rm_debugfs(d); - aoedisk_rm_sysfs(d); del_gendisk(d->gd); put_disk(d->gd); + blk_mq_free_tag_set(&d->tag_set); blk_cleanup_queue(d->blkq); } t = d->targets; @@ -464,6 +464,7 @@ aoedev_by_aoeaddr(ulong maj, int min, int do_alloc) d->ntargets = NTARGETS; INIT_WORK(&d->work, aoecmd_sleepwork); spin_lock_init(&d->lock); + INIT_LIST_HEAD(&d->rq_list); skb_queue_head_init(&d->skbpool); timer_setup(&d->timer, dummy_timer, 0); d->timer.expires = jiffies + HZ; diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c index dfb2c2622e5a..f88b4c26d422 100644 --- a/drivers/block/ataflop.c +++ b/drivers/block/ataflop.c @@ -66,13 +66,11 @@ #include <linux/fd.h> #include <linux/delay.h> #include <linux/init.h> -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/mutex.h> #include <linux/completion.h> #include <linux/wait.h> -#include <asm/atafd.h> -#include <asm/atafdreg.h> #include <asm/atariints.h> #include <asm/atari_stdma.h> #include <asm/atari_stram.h> @@ -83,7 +81,87 @@ static DEFINE_MUTEX(ataflop_mutex); static struct request *fd_request; -static int fdc_queue; + +/* + * WD1772 stuff + */ + +/* register codes */ + +#define FDCSELREG_STP (0x80) /* command/status register */ +#define FDCSELREG_TRA (0x82) /* track register */ +#define FDCSELREG_SEC (0x84) /* sector register */ +#define FDCSELREG_DTA (0x86) /* data register */ + +/* register names for FDC_READ/WRITE macros */ + +#define FDCREG_CMD 0 +#define FDCREG_STATUS 0 +#define FDCREG_TRACK 2 +#define FDCREG_SECTOR 4 +#define FDCREG_DATA 6 + +/* command opcodes */ + +#define FDCCMD_RESTORE (0x00) /* - */ +#define FDCCMD_SEEK (0x10) /* | */ +#define FDCCMD_STEP (0x20) /* | TYP 1 Commands */ +#define FDCCMD_STIN (0x40) /* | */ +#define FDCCMD_STOT (0x60) /* - */ +#define FDCCMD_RDSEC (0x80) /* - TYP 2 Commands */ +#define FDCCMD_WRSEC (0xa0) /* - " */ +#define FDCCMD_RDADR (0xc0) /* - */ +#define FDCCMD_RDTRA (0xe0) /* | TYP 3 Commands */ +#define FDCCMD_WRTRA (0xf0) /* - */ +#define FDCCMD_FORCI (0xd0) /* - TYP 4 Command */ + +/* command modifier bits */ + +#define FDCCMDADD_SR6 (0x00) /* step rate settings */ +#define FDCCMDADD_SR12 (0x01) +#define FDCCMDADD_SR2 (0x02) +#define FDCCMDADD_SR3 (0x03) +#define FDCCMDADD_V (0x04) /* verify */ +#define FDCCMDADD_H (0x08) /* wait for spin-up */ +#define FDCCMDADD_U (0x10) /* update track register */ +#define FDCCMDADD_M (0x10) /* multiple sector access */ +#define FDCCMDADD_E (0x04) /* head settling flag */ +#define FDCCMDADD_P (0x02) /* precompensation off */ +#define FDCCMDADD_A0 (0x01) /* DAM flag */ + +/* status register bits */ + +#define FDCSTAT_MOTORON (0x80) /* motor on */ +#define FDCSTAT_WPROT (0x40) /* write protected (FDCCMD_WR*) */ +#define FDCSTAT_SPINUP (0x20) /* motor speed stable (Type I) */ +#define FDCSTAT_DELDAM (0x20) /* sector has deleted DAM (Type II+III) */ +#define FDCSTAT_RECNF (0x10) /* record not found */ +#define FDCSTAT_CRC (0x08) /* CRC error */ +#define FDCSTAT_TR00 (0x04) /* Track 00 flag (Type I) */ +#define FDCSTAT_LOST (0x04) /* Lost Data (Type II+III) */ +#define FDCSTAT_IDX (0x02) /* Index status (Type I) */ +#define FDCSTAT_DRQ (0x02) /* DRQ status (Type II+III) */ +#define FDCSTAT_BUSY (0x01) /* FDC is busy */ + + +/* PSG Port A Bit Nr 0 .. Side Sel .. 0 -> Side 1 1 -> Side 2 */ +#define DSKSIDE (0x01) + +#define DSKDRVNONE (0x06) +#define DSKDRV0 (0x02) +#define DSKDRV1 (0x04) + +/* step rates */ +#define FDCSTEP_6 0x00 +#define FDCSTEP_12 0x01 +#define FDCSTEP_2 0x02 +#define FDCSTEP_3 0x03 + +struct atari_format_descr { + int track; /* to be formatted */ + int head; /* "" "" */ + int sect_offset; /* offset of first sector */ +}; /* Disk types: DD, HD, ED */ static struct atari_disk_type { @@ -221,6 +299,7 @@ static struct atari_floppy_struct { struct gendisk *disk; int ref; int type; + struct blk_mq_tag_set tag_set; } unit[FD_MAX_UNITS]; #define UD unit[drive] @@ -300,9 +379,6 @@ static int IsFormatting = 0, FormatError; static int UserSteprate[FD_MAX_UNITS] = { -1, -1 }; module_param_array(UserSteprate, int, NULL, 0); -/* Synchronization of FDC access. */ -static volatile int fdc_busy = 0; -static DECLARE_WAIT_QUEUE_HEAD(fdc_wait); static DECLARE_COMPLETION(format_wait); static unsigned long changed_floppies = 0xff, fake_change = 0; @@ -362,7 +438,6 @@ static void fd_times_out(struct timer_list *unused); static void finish_fdc( void ); static void finish_fdc_done( int dummy ); static void setup_req_params( int drive ); -static void redo_fd_request( void); static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long param); static void fd_probe( int drive ); @@ -380,8 +455,11 @@ static DEFINE_TIMER(fd_timer, check_change); static void fd_end_request_cur(blk_status_t err) { - if (!__blk_end_request_cur(fd_request, err)) + if (!blk_update_request(fd_request, err, + blk_rq_cur_bytes(fd_request))) { + __blk_mq_end_request(fd_request, err); fd_request = NULL; + } } static inline void start_motor_off_timer(void) @@ -627,7 +705,6 @@ static void fd_error( void ) if (SelectedDrive != -1) SUD.track = -1; } - redo_fd_request(); } @@ -645,14 +722,15 @@ static void fd_error( void ) static int do_format(int drive, int type, struct atari_format_descr *desc) { + struct request_queue *q = unit[drive].disk->queue; unsigned char *p; int sect, nsect; unsigned long flags; + int ret; - DPRINT(("do_format( dr=%d tr=%d he=%d offs=%d )\n", - drive, desc->track, desc->head, desc->sect_offset )); + blk_mq_freeze_queue(q); + blk_mq_quiesce_queue(q); - wait_event(fdc_wait, cmpxchg(&fdc_busy, 0, 1) == 0); local_irq_save(flags); stdma_lock(floppy_irq, NULL); atari_turnon_irq( IRQ_MFP_FDC ); /* should be already, just to be sure */ @@ -661,16 +739,16 @@ static int do_format(int drive, int type, struct atari_format_descr *desc) if (type) { if (--type >= NUM_DISK_MINORS || minor2disktype[type].drive_types > DriveType) { - redo_fd_request(); - return -EINVAL; + ret = -EINVAL; + goto out; } type = minor2disktype[type].index; UDT = &atari_disk_type[type]; } if (!UDT || desc->track >= UDT->blocks/UDT->spt/2 || desc->head >= 2) { - redo_fd_request(); - return -EINVAL; + ret = -EINVAL; + goto out; } nsect = UDT->spt; @@ -709,8 +787,11 @@ static int do_format(int drive, int type, struct atari_format_descr *desc) wait_for_completion(&format_wait); - redo_fd_request(); - return( FormatError ? -EIO : 0 ); + ret = FormatError ? -EIO : 0; +out: + blk_mq_unquiesce_queue(q); + blk_mq_unfreeze_queue(q); + return ret; } @@ -740,7 +821,6 @@ static void do_fd_action( int drive ) else { /* all sectors finished */ fd_end_request_cur(BLK_STS_OK); - redo_fd_request(); return; } } @@ -1145,7 +1225,6 @@ static void fd_rwsec_done1(int status) else { /* all sectors finished */ fd_end_request_cur(BLK_STS_OK); - redo_fd_request(); } return; @@ -1303,8 +1382,6 @@ static void finish_fdc_done( int dummy ) local_irq_save(flags); stdma_release(); - fdc_busy = 0; - wake_up( &fdc_wait ); local_irq_restore(flags); DPRINT(("finish_fdc() finished\n")); @@ -1394,59 +1471,34 @@ static void setup_req_params( int drive ) ReqTrack, ReqSector, (unsigned long)ReqData )); } -/* - * Round-robin between our available drives, doing one request from each - */ -static struct request *set_next_request(void) +static blk_status_t ataflop_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { - struct request_queue *q; - int old_pos = fdc_queue; - struct request *rq = NULL; - - do { - q = unit[fdc_queue].disk->queue; - if (++fdc_queue == FD_MAX_UNITS) - fdc_queue = 0; - if (q) { - rq = blk_fetch_request(q); - if (rq) { - rq->error_count = 0; - break; - } - } - } while (fdc_queue != old_pos); - - return rq; -} - + struct atari_floppy_struct *floppy = bd->rq->rq_disk->private_data; + int drive = floppy - unit; + int type = floppy->type; -static void redo_fd_request(void) -{ - int drive, type; - struct atari_floppy_struct *floppy; + spin_lock_irq(&ataflop_lock); + if (fd_request) { + spin_unlock_irq(&ataflop_lock); + return BLK_STS_DEV_RESOURCE; + } + if (!stdma_try_lock(floppy_irq, NULL)) { + spin_unlock_irq(&ataflop_lock); + return BLK_STS_RESOURCE; + } + fd_request = bd->rq; + blk_mq_start_request(fd_request); - DPRINT(("redo_fd_request: fd_request=%p dev=%s fd_request->sector=%ld\n", - fd_request, fd_request ? fd_request->rq_disk->disk_name : "", - fd_request ? blk_rq_pos(fd_request) : 0 )); + atari_disable_irq( IRQ_MFP_FDC ); IsFormatting = 0; -repeat: - if (!fd_request) { - fd_request = set_next_request(); - if (!fd_request) - goto the_end; - } - - floppy = fd_request->rq_disk->private_data; - drive = floppy - unit; - type = floppy->type; - if (!UD.connected) { /* drive not connected */ printk(KERN_ERR "Unknown Device: fd%d\n", drive ); fd_end_request_cur(BLK_STS_IOERR); - goto repeat; + goto out; } if (type == 0) { @@ -1462,23 +1514,18 @@ repeat: if (--type >= NUM_DISK_MINORS) { printk(KERN_WARNING "fd%d: invalid disk format", drive ); fd_end_request_cur(BLK_STS_IOERR); - goto repeat; + goto out; } if (minor2disktype[type].drive_types > DriveType) { printk(KERN_WARNING "fd%d: unsupported disk format", drive ); fd_end_request_cur(BLK_STS_IOERR); - goto repeat; + goto out; } type = minor2disktype[type].index; UDT = &atari_disk_type[type]; set_capacity(floppy->disk, UDT->blocks); UD.autoprobe = 0; } - - if (blk_rq_pos(fd_request) + 1 > UDT->blocks) { - fd_end_request_cur(BLK_STS_IOERR); - goto repeat; - } /* stop deselect timer */ del_timer( &motor_off_timer ); @@ -1490,22 +1537,13 @@ repeat: setup_req_params( drive ); do_fd_action( drive ); - return; - - the_end: - finish_fdc(); -} - - -void do_fd_request(struct request_queue * q) -{ - DPRINT(("do_fd_request for pid %d\n",current->pid)); - wait_event(fdc_wait, cmpxchg(&fdc_busy, 0, 1) == 0); - stdma_lock(floppy_irq, NULL); - - atari_disable_irq( IRQ_MFP_FDC ); - redo_fd_request(); + if (bd->last) + finish_fdc(); atari_enable_irq( IRQ_MFP_FDC ); + +out: + spin_unlock_irq(&ataflop_lock); + return BLK_STS_OK; } static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode, @@ -1583,7 +1621,6 @@ static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode, /* what if type > 0 here? Overwrite specified entry ? */ if (type) { /* refuse to re-set a predefined type for now */ - redo_fd_request(); return -EINVAL; } @@ -1651,10 +1688,8 @@ static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode, /* sanity check */ if (setprm.track != dtp->blocks/dtp->spt/2 || - setprm.head != 2) { - redo_fd_request(); + setprm.head != 2) return -EINVAL; - } UDT = dtp; set_capacity(floppy->disk, UDT->blocks); @@ -1910,6 +1945,10 @@ static const struct block_device_operations floppy_fops = { .revalidate_disk= floppy_revalidate, }; +static const struct blk_mq_ops ataflop_mq_ops = { + .queue_rq = ataflop_queue_rq, +}; + static struct kobject *floppy_find(dev_t dev, int *part, void *data) { int drive = *part & 3; @@ -1923,6 +1962,7 @@ static struct kobject *floppy_find(dev_t dev, int *part, void *data) static int __init atari_floppy_init (void) { int i; + int ret; if (!MACH_IS_ATARI) /* Amiga, Mac, ... don't have Atari-compatible floppy :-) */ @@ -1933,8 +1973,19 @@ static int __init atari_floppy_init (void) for (i = 0; i < FD_MAX_UNITS; i++) { unit[i].disk = alloc_disk(1); - if (!unit[i].disk) - goto Enomem; + if (!unit[i].disk) { + ret = -ENOMEM; + goto err; + } + + unit[i].disk->queue = blk_mq_init_sq_queue(&unit[i].tag_set, + &ataflop_mq_ops, 2, + BLK_MQ_F_SHOULD_MERGE); + if (IS_ERR(unit[i].disk->queue)) { + ret = PTR_ERR(unit[i].disk->queue); + unit[i].disk->queue = NULL; + goto err; + } } if (UseTrackbuffer < 0) @@ -1951,7 +2002,8 @@ static int __init atari_floppy_init (void) DMABuffer = atari_stram_alloc(BUFFER_SIZE+512, "ataflop"); if (!DMABuffer) { printk(KERN_ERR "atari_floppy_init: cannot get dma buffer\n"); - goto Enomem; + ret = -ENOMEM; + goto err; } TrackBuffer = DMABuffer + 512; PhysDMABuffer = atari_stram_to_phys(DMABuffer); @@ -1966,10 +2018,6 @@ static int __init atari_floppy_init (void) sprintf(unit[i].disk->disk_name, "fd%d", i); unit[i].disk->fops = &floppy_fops; unit[i].disk->private_data = &unit[i]; - unit[i].disk->queue = blk_init_queue(do_fd_request, - &ataflop_lock); - if (!unit[i].disk->queue) - goto Enomem; set_capacity(unit[i].disk, MAX_DISK_SIZE * 2); add_disk(unit[i].disk); } @@ -1983,17 +2031,23 @@ static int __init atari_floppy_init (void) config_types(); return 0; -Enomem: - while (i--) { - struct request_queue *q = unit[i].disk->queue; - put_disk(unit[i].disk); - if (q) - blk_cleanup_queue(q); - } +err: + do { + struct gendisk *disk = unit[i].disk; + + if (disk) { + if (disk->queue) { + blk_cleanup_queue(disk->queue); + disk->queue = NULL; + } + blk_mq_free_tag_set(&unit[i].tag_set); + put_disk(unit[i].disk); + } + } while (i--); unregister_blkdev(FLOPPY_MAJOR, "fd"); - return -ENOMEM; + return ret; } #ifndef MODULE @@ -2040,11 +2094,10 @@ static void __exit atari_floppy_exit(void) int i; blk_unregister_region(MKDEV(FLOPPY_MAJOR, 0), 256); for (i = 0; i < FD_MAX_UNITS; i++) { - struct request_queue *q = unit[i].disk->queue; - del_gendisk(unit[i].disk); + blk_cleanup_queue(unit[i].disk->queue); + blk_mq_free_tag_set(&unit[i].tag_set); put_disk(unit[i].disk); - blk_cleanup_queue(q); } unregister_blkdev(FLOPPY_MAJOR, "fd"); diff --git a/drivers/block/drbd/Kconfig b/drivers/block/drbd/Kconfig index 87aab6910d2d..52d885cdccb5 100644 --- a/drivers/block/drbd/Kconfig +++ b/drivers/block/drbd/Kconfig @@ -11,7 +11,6 @@ config BLK_DEV_DRBD depends on PROC_FS && INET select LRU_CACHE select LIBCRC32C - default n help NOTE: In order to authenticate connections you have to select diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h index e35a234b0a8f..1e47db57b9d2 100644 --- a/drivers/block/drbd/drbd_int.h +++ b/drivers/block/drbd/drbd_int.h @@ -429,7 +429,7 @@ enum { __EE_CALL_AL_COMPLETE_IO, __EE_MAY_SET_IN_SYNC, - /* is this a TRIM aka REQ_DISCARD? */ + /* is this a TRIM aka REQ_OP_DISCARD? */ __EE_IS_TRIM, /* In case a barrier failed, @@ -724,10 +724,10 @@ struct drbd_connection { struct list_head transfer_log; /* all requests not yet fully processed */ struct crypto_shash *cram_hmac_tfm; - struct crypto_ahash *integrity_tfm; /* checksums we compute, updates protected by connection->data->mutex */ - struct crypto_ahash *peer_integrity_tfm; /* checksums we verify, only accessed from receiver thread */ - struct crypto_ahash *csums_tfm; - struct crypto_ahash *verify_tfm; + struct crypto_shash *integrity_tfm; /* checksums we compute, updates protected by connection->data->mutex */ + struct crypto_shash *peer_integrity_tfm; /* checksums we verify, only accessed from receiver thread */ + struct crypto_shash *csums_tfm; + struct crypto_shash *verify_tfm; void *int_dig_in; void *int_dig_vv; @@ -1531,8 +1531,9 @@ static inline void ov_out_of_sync_print(struct drbd_device *device) } -extern void drbd_csum_bio(struct crypto_ahash *, struct bio *, void *); -extern void drbd_csum_ee(struct crypto_ahash *, struct drbd_peer_request *, void *); +extern void drbd_csum_bio(struct crypto_shash *, struct bio *, void *); +extern void drbd_csum_ee(struct crypto_shash *, struct drbd_peer_request *, + void *); /* worker callbacks */ extern int w_e_end_data_req(struct drbd_work *, int); extern int w_e_end_rsdata_req(struct drbd_work *, int); diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index ef8212a4b73e..55fd104f1ed4 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c @@ -1377,7 +1377,7 @@ void drbd_send_ack_dp(struct drbd_peer_device *peer_device, enum drbd_packet cmd struct p_data *dp, int data_size) { if (peer_device->connection->peer_integrity_tfm) - data_size -= crypto_ahash_digestsize(peer_device->connection->peer_integrity_tfm); + data_size -= crypto_shash_digestsize(peer_device->connection->peer_integrity_tfm); _drbd_send_ack(peer_device, cmd, dp->sector, cpu_to_be32(data_size), dp->block_id); } @@ -1673,7 +1673,7 @@ static u32 bio_flags_to_wire(struct drbd_connection *connection, return bio->bi_opf & REQ_SYNC ? DP_RW_SYNC : 0; } -/* Used to send write or TRIM aka REQ_DISCARD requests +/* Used to send write or TRIM aka REQ_OP_DISCARD requests * R_PRIMARY -> Peer (P_DATA, P_TRIM) */ int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request *req) @@ -1690,7 +1690,7 @@ int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request * sock = &peer_device->connection->data; p = drbd_prepare_command(peer_device, sock); digest_size = peer_device->connection->integrity_tfm ? - crypto_ahash_digestsize(peer_device->connection->integrity_tfm) : 0; + crypto_shash_digestsize(peer_device->connection->integrity_tfm) : 0; if (!p) return -EIO; @@ -1796,7 +1796,7 @@ int drbd_send_block(struct drbd_peer_device *peer_device, enum drbd_packet cmd, p = drbd_prepare_command(peer_device, sock); digest_size = peer_device->connection->integrity_tfm ? - crypto_ahash_digestsize(peer_device->connection->integrity_tfm) : 0; + crypto_shash_digestsize(peer_device->connection->integrity_tfm) : 0; if (!p) return -EIO; @@ -2557,11 +2557,11 @@ void conn_free_crypto(struct drbd_connection *connection) { drbd_free_sock(connection); - crypto_free_ahash(connection->csums_tfm); - crypto_free_ahash(connection->verify_tfm); + crypto_free_shash(connection->csums_tfm); + crypto_free_shash(connection->verify_tfm); crypto_free_shash(connection->cram_hmac_tfm); - crypto_free_ahash(connection->integrity_tfm); - crypto_free_ahash(connection->peer_integrity_tfm); + crypto_free_shash(connection->integrity_tfm); + crypto_free_shash(connection->peer_integrity_tfm); kfree(connection->int_dig_in); kfree(connection->int_dig_vv); diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c index b4f02768ba47..d15703b1ffe8 100644 --- a/drivers/block/drbd/drbd_nl.c +++ b/drivers/block/drbd/drbd_nl.c @@ -2303,10 +2303,10 @@ check_net_options(struct drbd_connection *connection, struct net_conf *new_net_c } struct crypto { - struct crypto_ahash *verify_tfm; - struct crypto_ahash *csums_tfm; + struct crypto_shash *verify_tfm; + struct crypto_shash *csums_tfm; struct crypto_shash *cram_hmac_tfm; - struct crypto_ahash *integrity_tfm; + struct crypto_shash *integrity_tfm; }; static int @@ -2324,36 +2324,21 @@ alloc_shash(struct crypto_shash **tfm, char *tfm_name, int err_alg) return NO_ERROR; } -static int -alloc_ahash(struct crypto_ahash **tfm, char *tfm_name, int err_alg) -{ - if (!tfm_name[0]) - return NO_ERROR; - - *tfm = crypto_alloc_ahash(tfm_name, 0, CRYPTO_ALG_ASYNC); - if (IS_ERR(*tfm)) { - *tfm = NULL; - return err_alg; - } - - return NO_ERROR; -} - static enum drbd_ret_code alloc_crypto(struct crypto *crypto, struct net_conf *new_net_conf) { char hmac_name[CRYPTO_MAX_ALG_NAME]; enum drbd_ret_code rv; - rv = alloc_ahash(&crypto->csums_tfm, new_net_conf->csums_alg, + rv = alloc_shash(&crypto->csums_tfm, new_net_conf->csums_alg, ERR_CSUMS_ALG); if (rv != NO_ERROR) return rv; - rv = alloc_ahash(&crypto->verify_tfm, new_net_conf->verify_alg, + rv = alloc_shash(&crypto->verify_tfm, new_net_conf->verify_alg, ERR_VERIFY_ALG); if (rv != NO_ERROR) return rv; - rv = alloc_ahash(&crypto->integrity_tfm, new_net_conf->integrity_alg, + rv = alloc_shash(&crypto->integrity_tfm, new_net_conf->integrity_alg, ERR_INTEGRITY_ALG); if (rv != NO_ERROR) return rv; @@ -2371,9 +2356,9 @@ alloc_crypto(struct crypto *crypto, struct net_conf *new_net_conf) static void free_crypto(struct crypto *crypto) { crypto_free_shash(crypto->cram_hmac_tfm); - crypto_free_ahash(crypto->integrity_tfm); - crypto_free_ahash(crypto->csums_tfm); - crypto_free_ahash(crypto->verify_tfm); + crypto_free_shash(crypto->integrity_tfm); + crypto_free_shash(crypto->csums_tfm); + crypto_free_shash(crypto->verify_tfm); } int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info) @@ -2450,17 +2435,17 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info) rcu_assign_pointer(connection->net_conf, new_net_conf); if (!rsr) { - crypto_free_ahash(connection->csums_tfm); + crypto_free_shash(connection->csums_tfm); connection->csums_tfm = crypto.csums_tfm; crypto.csums_tfm = NULL; } if (!ovr) { - crypto_free_ahash(connection->verify_tfm); + crypto_free_shash(connection->verify_tfm); connection->verify_tfm = crypto.verify_tfm; crypto.verify_tfm = NULL; } - crypto_free_ahash(connection->integrity_tfm); + crypto_free_shash(connection->integrity_tfm); connection->integrity_tfm = crypto.integrity_tfm; if (connection->cstate >= C_WF_REPORT_PARAMS && connection->agreed_pro_version >= 100) /* Do this without trying to take connection->data.mutex again. */ diff --git a/drivers/block/drbd/drbd_protocol.h b/drivers/block/drbd/drbd_protocol.h index c3081f93051c..48dabbb21e11 100644 --- a/drivers/block/drbd/drbd_protocol.h +++ b/drivers/block/drbd/drbd_protocol.h @@ -57,7 +57,7 @@ enum drbd_packet { P_PROTOCOL_UPDATE = 0x2d, /* data sock: is used in established connections */ /* 0x2e to 0x30 reserved, used in drbd 9 */ - /* REQ_DISCARD. We used "discard" in different contexts before, + /* REQ_OP_DISCARD. We used "discard" in different contexts before, * which is why I chose TRIM here, to disambiguate. */ P_TRIM = 0x31, @@ -126,7 +126,7 @@ struct p_header100 { #define DP_UNPLUG 8 /* not used anymore */ #define DP_FUA 16 /* equals REQ_FUA */ #define DP_FLUSH 32 /* equals REQ_PREFLUSH */ -#define DP_DISCARD 64 /* equals REQ_DISCARD */ +#define DP_DISCARD 64 /* equals REQ_OP_DISCARD */ #define DP_SEND_RECEIVE_ACK 128 /* This is a proto B write request */ #define DP_SEND_WRITE_ACK 256 /* This is a proto C write request */ #define DP_WSAME 512 /* equiv. REQ_WRITE_SAME */ diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c index 75f6b47169e6..fc67fd853375 100644 --- a/drivers/block/drbd/drbd_receiver.c +++ b/drivers/block/drbd/drbd_receiver.c @@ -1732,7 +1732,7 @@ static int receive_Barrier(struct drbd_connection *connection, struct packet_inf } /* quick wrapper in case payload size != request_size (write same) */ -static void drbd_csum_ee_size(struct crypto_ahash *h, +static void drbd_csum_ee_size(struct crypto_shash *h, struct drbd_peer_request *r, void *d, unsigned int payload_size) { @@ -1769,7 +1769,7 @@ read_in_block(struct drbd_peer_device *peer_device, u64 id, sector_t sector, digest_size = 0; if (!trim && peer_device->connection->peer_integrity_tfm) { - digest_size = crypto_ahash_digestsize(peer_device->connection->peer_integrity_tfm); + digest_size = crypto_shash_digestsize(peer_device->connection->peer_integrity_tfm); /* * FIXME: Receive the incoming digest into the receive buffer * here, together with its struct p_data? @@ -1905,7 +1905,7 @@ static int recv_dless_read(struct drbd_peer_device *peer_device, struct drbd_req digest_size = 0; if (peer_device->connection->peer_integrity_tfm) { - digest_size = crypto_ahash_digestsize(peer_device->connection->peer_integrity_tfm); + digest_size = crypto_shash_digestsize(peer_device->connection->peer_integrity_tfm); err = drbd_recv_all_warn(peer_device->connection, dig_in, digest_size); if (err) return err; @@ -3542,7 +3542,7 @@ static int receive_protocol(struct drbd_connection *connection, struct packet_in int p_proto, p_discard_my_data, p_two_primaries, cf; struct net_conf *nc, *old_net_conf, *new_net_conf = NULL; char integrity_alg[SHARED_SECRET_MAX] = ""; - struct crypto_ahash *peer_integrity_tfm = NULL; + struct crypto_shash *peer_integrity_tfm = NULL; void *int_dig_in = NULL, *int_dig_vv = NULL; p_proto = be32_to_cpu(p->protocol); @@ -3623,7 +3623,7 @@ static int receive_protocol(struct drbd_connection *connection, struct packet_in * change. */ - peer_integrity_tfm = crypto_alloc_ahash(integrity_alg, 0, CRYPTO_ALG_ASYNC); + peer_integrity_tfm = crypto_alloc_shash(integrity_alg, 0, CRYPTO_ALG_ASYNC); if (IS_ERR(peer_integrity_tfm)) { peer_integrity_tfm = NULL; drbd_err(connection, "peer data-integrity-alg %s not supported\n", @@ -3631,7 +3631,7 @@ static int receive_protocol(struct drbd_connection *connection, struct packet_in goto disconnect; } - hash_size = crypto_ahash_digestsize(peer_integrity_tfm); + hash_size = crypto_shash_digestsize(peer_integrity_tfm); int_dig_in = kmalloc(hash_size, GFP_KERNEL); int_dig_vv = kmalloc(hash_size, GFP_KERNEL); if (!(int_dig_in && int_dig_vv)) { @@ -3661,7 +3661,7 @@ static int receive_protocol(struct drbd_connection *connection, struct packet_in mutex_unlock(&connection->resource->conf_update); mutex_unlock(&connection->data.mutex); - crypto_free_ahash(connection->peer_integrity_tfm); + crypto_free_shash(connection->peer_integrity_tfm); kfree(connection->int_dig_in); kfree(connection->int_dig_vv); connection->peer_integrity_tfm = peer_integrity_tfm; @@ -3679,7 +3679,7 @@ static int receive_protocol(struct drbd_connection *connection, struct packet_in disconnect_rcu_unlock: rcu_read_unlock(); disconnect: - crypto_free_ahash(peer_integrity_tfm); + crypto_free_shash(peer_integrity_tfm); kfree(int_dig_in); kfree(int_dig_vv); conn_request_state(connection, NS(conn, C_DISCONNECTING), CS_HARD); @@ -3691,15 +3691,16 @@ disconnect: * return: NULL (alg name was "") * ERR_PTR(error) if something goes wrong * or the crypto hash ptr, if it worked out ok. */ -static struct crypto_ahash *drbd_crypto_alloc_digest_safe(const struct drbd_device *device, +static struct crypto_shash *drbd_crypto_alloc_digest_safe( + const struct drbd_device *device, const char *alg, const char *name) { - struct crypto_ahash *tfm; + struct crypto_shash *tfm; if (!alg[0]) return NULL; - tfm = crypto_alloc_ahash(alg, 0, CRYPTO_ALG_ASYNC); + tfm = crypto_alloc_shash(alg, 0, 0); if (IS_ERR(tfm)) { drbd_err(device, "Can not allocate \"%s\" as %s (reason: %ld)\n", alg, name, PTR_ERR(tfm)); @@ -3752,8 +3753,8 @@ static int receive_SyncParam(struct drbd_connection *connection, struct packet_i struct drbd_device *device; struct p_rs_param_95 *p; unsigned int header_size, data_size, exp_max_sz; - struct crypto_ahash *verify_tfm = NULL; - struct crypto_ahash *csums_tfm = NULL; + struct crypto_shash *verify_tfm = NULL; + struct crypto_shash *csums_tfm = NULL; struct net_conf *old_net_conf, *new_net_conf = NULL; struct disk_conf *old_disk_conf = NULL, *new_disk_conf = NULL; const int apv = connection->agreed_pro_version; @@ -3900,14 +3901,14 @@ static int receive_SyncParam(struct drbd_connection *connection, struct packet_i if (verify_tfm) { strcpy(new_net_conf->verify_alg, p->verify_alg); new_net_conf->verify_alg_len = strlen(p->verify_alg) + 1; - crypto_free_ahash(peer_device->connection->verify_tfm); + crypto_free_shash(peer_device->connection->verify_tfm); peer_device->connection->verify_tfm = verify_tfm; drbd_info(device, "using verify-alg: \"%s\"\n", p->verify_alg); } if (csums_tfm) { strcpy(new_net_conf->csums_alg, p->csums_alg); new_net_conf->csums_alg_len = strlen(p->csums_alg) + 1; - crypto_free_ahash(peer_device->connection->csums_tfm); + crypto_free_shash(peer_device->connection->csums_tfm); peer_device->connection->csums_tfm = csums_tfm; drbd_info(device, "using csums-alg: \"%s\"\n", p->csums_alg); } @@ -3951,9 +3952,9 @@ disconnect: mutex_unlock(&connection->resource->conf_update); /* just for completeness: actually not needed, * as this is not reached if csums_tfm was ok. */ - crypto_free_ahash(csums_tfm); + crypto_free_shash(csums_tfm); /* but free the verify_tfm again, if csums_tfm did not work out */ - crypto_free_ahash(verify_tfm); + crypto_free_shash(verify_tfm); conn_request_state(peer_device->connection, NS(conn, C_DISCONNECTING), CS_HARD); return -EIO; } diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c index 19cac36e9737..1c4da17e902e 100644 --- a/drivers/block/drbd/drbd_req.c +++ b/drivers/block/drbd/drbd_req.c @@ -650,7 +650,7 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what, case DISCARD_COMPLETED_NOTSUPP: case DISCARD_COMPLETED_WITH_ERROR: /* I'd rather not detach from local disk just because it - * failed a REQ_DISCARD. */ + * failed a REQ_OP_DISCARD. */ mod_rq_state(req, m, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED); break; diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c index b8f77e83d456..99255d0c9e2f 100644 --- a/drivers/block/drbd/drbd_worker.c +++ b/drivers/block/drbd/drbd_worker.c @@ -152,7 +152,7 @@ void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req) __releases(l do_wake = list_empty(block_id == ID_SYNCER ? &device->sync_ee : &device->active_ee); - /* FIXME do we want to detach for failed REQ_DISCARD? + /* FIXME do we want to detach for failed REQ_OP_DISCARD? * ((peer_req->flags & (EE_WAS_ERROR|EE_IS_TRIM)) == EE_WAS_ERROR) */ if (peer_req->flags & EE_WAS_ERROR) __drbd_chk_io_error(device, DRBD_WRITE_ERROR); @@ -295,60 +295,61 @@ void drbd_request_endio(struct bio *bio) complete_master_bio(device, &m); } -void drbd_csum_ee(struct crypto_ahash *tfm, struct drbd_peer_request *peer_req, void *digest) +void drbd_csum_ee(struct crypto_shash *tfm, struct drbd_peer_request *peer_req, void *digest) { - AHASH_REQUEST_ON_STACK(req, tfm); - struct scatterlist sg; + SHASH_DESC_ON_STACK(desc, tfm); struct page *page = peer_req->pages; struct page *tmp; unsigned len; + void *src; - ahash_request_set_tfm(req, tfm); - ahash_request_set_callback(req, 0, NULL, NULL); + desc->tfm = tfm; + desc->flags = 0; - sg_init_table(&sg, 1); - crypto_ahash_init(req); + crypto_shash_init(desc); + src = kmap_atomic(page); while ((tmp = page_chain_next(page))) { /* all but the last page will be fully used */ - sg_set_page(&sg, page, PAGE_SIZE, 0); - ahash_request_set_crypt(req, &sg, NULL, sg.length); - crypto_ahash_update(req); + crypto_shash_update(desc, src, PAGE_SIZE); + kunmap_atomic(src); page = tmp; + src = kmap_atomic(page); } /* and now the last, possibly only partially used page */ len = peer_req->i.size & (PAGE_SIZE - 1); - sg_set_page(&sg, page, len ?: PAGE_SIZE, 0); - ahash_request_set_crypt(req, &sg, digest, sg.length); - crypto_ahash_finup(req); - ahash_request_zero(req); + crypto_shash_update(desc, src, len ?: PAGE_SIZE); + kunmap_atomic(src); + + crypto_shash_final(desc, digest); + shash_desc_zero(desc); } -void drbd_csum_bio(struct crypto_ahash *tfm, struct bio *bio, void *digest) +void drbd_csum_bio(struct crypto_shash *tfm, struct bio *bio, void *digest) { - AHASH_REQUEST_ON_STACK(req, tfm); - struct scatterlist sg; + SHASH_DESC_ON_STACK(desc, tfm); struct bio_vec bvec; struct bvec_iter iter; - ahash_request_set_tfm(req, tfm); - ahash_request_set_callback(req, 0, NULL, NULL); + desc->tfm = tfm; + desc->flags = 0; - sg_init_table(&sg, 1); - crypto_ahash_init(req); + crypto_shash_init(desc); bio_for_each_segment(bvec, bio, iter) { - sg_set_page(&sg, bvec.bv_page, bvec.bv_len, bvec.bv_offset); - ahash_request_set_crypt(req, &sg, NULL, sg.length); - crypto_ahash_update(req); + u8 *src; + + src = kmap_atomic(bvec.bv_page); + crypto_shash_update(desc, src + bvec.bv_offset, bvec.bv_len); + kunmap_atomic(src); + /* REQ_OP_WRITE_SAME has only one segment, * checksum the payload only once. */ if (bio_op(bio) == REQ_OP_WRITE_SAME) break; } - ahash_request_set_crypt(req, NULL, digest, 0); - crypto_ahash_final(req); - ahash_request_zero(req); + crypto_shash_final(desc, digest); + shash_desc_zero(desc); } /* MAYBE merge common code with w_e_end_ov_req */ @@ -367,7 +368,7 @@ static int w_e_send_csum(struct drbd_work *w, int cancel) if (unlikely((peer_req->flags & EE_WAS_ERROR) != 0)) goto out; - digest_size = crypto_ahash_digestsize(peer_device->connection->csums_tfm); + digest_size = crypto_shash_digestsize(peer_device->connection->csums_tfm); digest = kmalloc(digest_size, GFP_NOIO); if (digest) { sector_t sector = peer_req->i.sector; @@ -1205,7 +1206,7 @@ int w_e_end_csum_rs_req(struct drbd_work *w, int cancel) * a real fix would be much more involved, * introducing more locking mechanisms */ if (peer_device->connection->csums_tfm) { - digest_size = crypto_ahash_digestsize(peer_device->connection->csums_tfm); + digest_size = crypto_shash_digestsize(peer_device->connection->csums_tfm); D_ASSERT(device, digest_size == di->digest_size); digest = kmalloc(digest_size, GFP_NOIO); } @@ -1255,7 +1256,7 @@ int w_e_end_ov_req(struct drbd_work *w, int cancel) if (unlikely(cancel)) goto out; - digest_size = crypto_ahash_digestsize(peer_device->connection->verify_tfm); + digest_size = crypto_shash_digestsize(peer_device->connection->verify_tfm); digest = kmalloc(digest_size, GFP_NOIO); if (!digest) { err = 1; /* terminate the connection in case the allocation failed */ @@ -1327,7 +1328,7 @@ int w_e_end_ov_reply(struct drbd_work *w, int cancel) di = peer_req->digest; if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) { - digest_size = crypto_ahash_digestsize(peer_device->connection->verify_tfm); + digest_size = crypto_shash_digestsize(peer_device->connection->verify_tfm); digest = kmalloc(digest_size, GFP_NOIO); if (digest) { drbd_csum_ee(peer_device->connection->verify_tfm, peer_req, digest); diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index f2b6f4da1034..a8cfa011c284 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -252,13 +252,13 @@ static int allowed_drive_mask = 0x33; static int irqdma_allocated; -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/blkpg.h> #include <linux/cdrom.h> /* for the compatibility eject ioctl */ #include <linux/completion.h> +static LIST_HEAD(floppy_reqs); static struct request *current_req; -static void do_fd_request(struct request_queue *q); static int set_next_request(void); #ifndef fd_get_dma_residue @@ -414,10 +414,10 @@ static struct floppy_drive_struct drive_state[N_DRIVE]; static struct floppy_write_errors write_errors[N_DRIVE]; static struct timer_list motor_off_timer[N_DRIVE]; static struct gendisk *disks[N_DRIVE]; +static struct blk_mq_tag_set tag_sets[N_DRIVE]; static struct block_device *opened_bdev[N_DRIVE]; static DEFINE_MUTEX(open_lock); static struct floppy_raw_cmd *raw_cmd, default_raw_cmd; -static int fdc_queue; /* * This struct defines the different floppy types. @@ -2216,8 +2216,9 @@ static void floppy_end_request(struct request *req, blk_status_t error) /* current_count_sectors can be zero if transfer failed */ if (error) nr_sectors = blk_rq_cur_sectors(req); - if (__blk_end_request(req, error, nr_sectors << 9)) + if (blk_update_request(req, error, nr_sectors << 9)) return; + __blk_mq_end_request(req, error); /* We're done with the request */ floppy_off(drive); @@ -2797,27 +2798,14 @@ static int make_raw_rw_request(void) return 2; } -/* - * Round-robin between our available drives, doing one request from each - */ static int set_next_request(void) { - struct request_queue *q; - int old_pos = fdc_queue; - - do { - q = disks[fdc_queue]->queue; - if (++fdc_queue == N_DRIVE) - fdc_queue = 0; - if (q) { - current_req = blk_fetch_request(q); - if (current_req) { - current_req->error_count = 0; - break; - } - } - } while (fdc_queue != old_pos); - + current_req = list_first_entry_or_null(&floppy_reqs, struct request, + queuelist); + if (current_req) { + current_req->error_count = 0; + list_del_init(¤t_req->queuelist); + } return current_req != NULL; } @@ -2901,29 +2889,38 @@ static void process_fd_request(void) schedule_bh(redo_fd_request); } -static void do_fd_request(struct request_queue *q) +static blk_status_t floppy_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { + blk_mq_start_request(bd->rq); + if (WARN(max_buffer_sectors == 0, "VFS: %s called on non-open device\n", __func__)) - return; + return BLK_STS_IOERR; if (WARN(atomic_read(&usage_count) == 0, "warning: usage count=0, current_req=%p sect=%ld flags=%llx\n", current_req, (long)blk_rq_pos(current_req), (unsigned long long) current_req->cmd_flags)) - return; + return BLK_STS_IOERR; + + spin_lock_irq(&floppy_lock); + list_add_tail(&bd->rq->queuelist, &floppy_reqs); + spin_unlock_irq(&floppy_lock); if (test_and_set_bit(0, &fdc_busy)) { /* fdc busy, this new request will be treated when the current one is done */ is_alive(__func__, "old request running"); - return; + return BLK_STS_OK; } + command_status = FD_COMMAND_NONE; __reschedule_timeout(MAXTIMEOUT, "fd_request"); set_fdc(0); process_fd_request(); is_alive(__func__, ""); + return BLK_STS_OK; } static const struct cont_t poll_cont = { @@ -4486,6 +4483,10 @@ static struct platform_driver floppy_driver = { }, }; +static const struct blk_mq_ops floppy_mq_ops = { + .queue_rq = floppy_queue_rq, +}; + static struct platform_device floppy_device[N_DRIVE]; static bool floppy_available(int drive) @@ -4533,9 +4534,12 @@ static int __init do_floppy_init(void) goto out_put_disk; } - disks[drive]->queue = blk_init_queue(do_fd_request, &floppy_lock); - if (!disks[drive]->queue) { - err = -ENOMEM; + disks[drive]->queue = blk_mq_init_sq_queue(&tag_sets[drive], + &floppy_mq_ops, 2, + BLK_MQ_F_SHOULD_MERGE); + if (IS_ERR(disks[drive]->queue)) { + err = PTR_ERR(disks[drive]->queue); + disks[drive]->queue = NULL; goto out_put_disk; } @@ -4679,7 +4683,7 @@ static int __init do_floppy_init(void) /* to be cleaned up... */ disks[drive]->private_data = (void *)(long)drive; disks[drive]->flags |= GENHD_FL_REMOVABLE; - device_add_disk(&floppy_device[drive].dev, disks[drive]); + device_add_disk(&floppy_device[drive].dev, disks[drive], NULL); } return 0; @@ -4708,6 +4712,7 @@ out_put_disk: del_timer_sync(&motor_off_timer[drive]); blk_cleanup_queue(disks[drive]->queue); disks[drive]->queue = NULL; + blk_mq_free_tag_set(&tag_sets[drive]); } put_disk(disks[drive]); } @@ -4935,6 +4940,7 @@ static void __exit floppy_module_exit(void) platform_device_unregister(&floppy_device[drive]); } blk_cleanup_queue(disks[drive]->queue); + blk_mq_free_tag_set(&tag_sets[drive]); /* * These disks have not called add_disk(). Don't put down diff --git a/drivers/block/loop.c b/drivers/block/loop.c index ea9debf59b22..abad6d15f956 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -77,6 +77,7 @@ #include <linux/falloc.h> #include <linux/uio.h> #include <linux/ioprio.h> +#include <linux/blk-cgroup.h> #include "loop.h" @@ -1760,8 +1761,8 @@ static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx, /* always use the first bio's css */ #ifdef CONFIG_BLK_CGROUP - if (cmd->use_aio && rq->bio && rq->bio->bi_css) { - cmd->css = rq->bio->bi_css; + if (cmd->use_aio && rq->bio && rq->bio->bi_blkg) { + cmd->css = &bio_blkcg(rq->bio)->css; css_get(cmd->css); } else #endif diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index d0666f5ce003..dfc8de6ce525 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -1862,11 +1862,9 @@ static int exec_drive_taskfile(struct driver_data *dd, if (IS_ERR(outbuf)) return PTR_ERR(outbuf); - outbuf_dma = pci_map_single(dd->pdev, - outbuf, - taskout, - DMA_TO_DEVICE); - if (pci_dma_mapping_error(dd->pdev, outbuf_dma)) { + outbuf_dma = dma_map_single(&dd->pdev->dev, outbuf, + taskout, DMA_TO_DEVICE); + if (dma_mapping_error(&dd->pdev->dev, outbuf_dma)) { err = -ENOMEM; goto abort; } @@ -1880,10 +1878,9 @@ static int exec_drive_taskfile(struct driver_data *dd, inbuf = NULL; goto abort; } - inbuf_dma = pci_map_single(dd->pdev, - inbuf, - taskin, DMA_FROM_DEVICE); - if (pci_dma_mapping_error(dd->pdev, inbuf_dma)) { + inbuf_dma = dma_map_single(&dd->pdev->dev, inbuf, + taskin, DMA_FROM_DEVICE); + if (dma_mapping_error(&dd->pdev->dev, inbuf_dma)) { err = -ENOMEM; goto abort; } @@ -2002,11 +1999,11 @@ static int exec_drive_taskfile(struct driver_data *dd, /* reclaim the DMA buffers.*/ if (inbuf_dma) - pci_unmap_single(dd->pdev, inbuf_dma, - taskin, DMA_FROM_DEVICE); + dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin, + DMA_FROM_DEVICE); if (outbuf_dma) - pci_unmap_single(dd->pdev, outbuf_dma, - taskout, DMA_TO_DEVICE); + dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout, + DMA_TO_DEVICE); inbuf_dma = 0; outbuf_dma = 0; @@ -2053,11 +2050,11 @@ static int exec_drive_taskfile(struct driver_data *dd, } abort: if (inbuf_dma) - pci_unmap_single(dd->pdev, inbuf_dma, - taskin, DMA_FROM_DEVICE); + dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin, + DMA_FROM_DEVICE); if (outbuf_dma) - pci_unmap_single(dd->pdev, outbuf_dma, - taskout, DMA_TO_DEVICE); + dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout, + DMA_TO_DEVICE); kfree(outbuf); kfree(inbuf); @@ -3861,7 +3858,7 @@ skip_create_disk: set_capacity(dd->disk, capacity); /* Enable the block device and add it to /dev */ - device_add_disk(&dd->pdev->dev, dd->disk); + device_add_disk(&dd->pdev->dev, dd->disk, NULL); dd->bdev = bdget_disk(dd->disk, 0); /* @@ -4216,18 +4213,10 @@ static int mtip_pci_probe(struct pci_dev *pdev, goto iomap_err; } - if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { - rv = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); - - if (rv) { - rv = pci_set_consistent_dma_mask(pdev, - DMA_BIT_MASK(32)); - if (rv) { - dev_warn(&pdev->dev, - "64-bit DMA enable failed\n"); - goto setmask_err; - } - } + rv = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (rv) { + dev_warn(&pdev->dev, "64-bit DMA enable failed\n"); + goto setmask_err; } /* Copy the info we may need later into the private data structure. */ diff --git a/drivers/block/null_blk_main.c b/drivers/block/null_blk_main.c index 093b614d6524..e94591021682 100644 --- a/drivers/block/null_blk_main.c +++ b/drivers/block/null_blk_main.c @@ -606,20 +606,12 @@ static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait) static void end_cmd(struct nullb_cmd *cmd) { - struct request_queue *q = NULL; int queue_mode = cmd->nq->dev->queue_mode; - if (cmd->rq) - q = cmd->rq->q; - switch (queue_mode) { case NULL_Q_MQ: blk_mq_end_request(cmd->rq, cmd->error); return; - case NULL_Q_RQ: - INIT_LIST_HEAD(&cmd->rq->queuelist); - blk_end_request_all(cmd->rq, cmd->error); - break; case NULL_Q_BIO: cmd->bio->bi_status = cmd->error; bio_endio(cmd->bio); @@ -627,15 +619,6 @@ static void end_cmd(struct nullb_cmd *cmd) } free_cmd(cmd); - - /* Restart queue if needed, as we are freeing a tag */ - if (queue_mode == NULL_Q_RQ && blk_queue_stopped(q)) { - unsigned long flags; - - spin_lock_irqsave(q->queue_lock, flags); - blk_start_queue_async(q); - spin_unlock_irqrestore(q->queue_lock, flags); - } } static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer) @@ -1136,25 +1119,14 @@ static void null_stop_queue(struct nullb *nullb) if (nullb->dev->queue_mode == NULL_Q_MQ) blk_mq_stop_hw_queues(q); - else { - spin_lock_irq(q->queue_lock); - blk_stop_queue(q); - spin_unlock_irq(q->queue_lock); - } } static void null_restart_queue_async(struct nullb *nullb) { struct request_queue *q = nullb->q; - unsigned long flags; if (nullb->dev->queue_mode == NULL_Q_MQ) blk_mq_start_stopped_hw_queues(q, true); - else { - spin_lock_irqsave(q->queue_lock, flags); - blk_start_queue_async(q); - spin_unlock_irqrestore(q->queue_lock, flags); - } } static bool cmd_report_zone(struct nullb *nullb, struct nullb_cmd *cmd) @@ -1197,17 +1169,8 @@ static blk_status_t null_handle_cmd(struct nullb_cmd *cmd) /* race with timer */ if (atomic_long_read(&nullb->cur_bytes) > 0) null_restart_queue_async(nullb); - if (dev->queue_mode == NULL_Q_RQ) { - struct request_queue *q = nullb->q; - - spin_lock_irq(q->queue_lock); - rq->rq_flags |= RQF_DONTPREP; - blk_requeue_request(q, rq); - spin_unlock_irq(q->queue_lock); - return BLK_STS_OK; - } else - /* requeue request */ - return BLK_STS_DEV_RESOURCE; + /* requeue request */ + return BLK_STS_DEV_RESOURCE; } } @@ -1278,9 +1241,6 @@ out: case NULL_Q_MQ: blk_mq_complete_request(cmd->rq); break; - case NULL_Q_RQ: - blk_complete_request(cmd->rq); - break; case NULL_Q_BIO: /* * XXX: no proper submitting cpu information available. @@ -1349,30 +1309,6 @@ static blk_qc_t null_queue_bio(struct request_queue *q, struct bio *bio) return BLK_QC_T_NONE; } -static enum blk_eh_timer_return null_rq_timed_out_fn(struct request *rq) -{ - pr_info("null: rq %p timed out\n", rq); - __blk_complete_request(rq); - return BLK_EH_DONE; -} - -static int null_rq_prep_fn(struct request_queue *q, struct request *req) -{ - struct nullb *nullb = q->queuedata; - struct nullb_queue *nq = nullb_to_queue(nullb); - struct nullb_cmd *cmd; - - cmd = alloc_cmd(nq, 0); - if (cmd) { - cmd->rq = req; - req->special = cmd; - return BLKPREP_OK; - } - blk_stop_queue(q); - - return BLKPREP_DEFER; -} - static bool should_timeout_request(struct request *rq) { #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION @@ -1391,27 +1327,6 @@ static bool should_requeue_request(struct request *rq) return false; } -static void null_request_fn(struct request_queue *q) -{ - struct request *rq; - - while ((rq = blk_fetch_request(q)) != NULL) { - struct nullb_cmd *cmd = rq->special; - - /* just ignore the request */ - if (should_timeout_request(rq)) - continue; - if (should_requeue_request(rq)) { - blk_requeue_request(q, rq); - continue; - } - - spin_unlock_irq(q->queue_lock); - null_handle_cmd(cmd); - spin_lock_irq(q->queue_lock); - } -} - static enum blk_eh_timer_return null_timeout_rq(struct request *rq, bool res) { pr_info("null: rq %p timed out\n", rq); @@ -1766,24 +1681,6 @@ static int null_add_dev(struct nullb_device *dev) rv = init_driver_queues(nullb); if (rv) goto out_cleanup_blk_queue; - } else { - nullb->q = blk_init_queue_node(null_request_fn, &nullb->lock, - dev->home_node); - if (!nullb->q) { - rv = -ENOMEM; - goto out_cleanup_queues; - } - - if (!null_setup_fault()) - goto out_cleanup_blk_queue; - - blk_queue_prep_rq(nullb->q, null_rq_prep_fn); - blk_queue_softirq_done(nullb->q, null_softirq_done_fn); - blk_queue_rq_timed_out(nullb->q, null_rq_timed_out_fn); - nullb->q->rq_timeout = 5 * HZ; - rv = init_driver_queues(nullb); - if (rv) - goto out_cleanup_blk_queue; } if (dev->mbps) { @@ -1865,6 +1762,10 @@ static int __init null_init(void) return -EINVAL; } + if (g_queue_mode == NULL_Q_RQ) { + pr_err("null_blk: legacy IO path no longer available\n"); + return -EINVAL; + } if (g_queue_mode == NULL_Q_MQ && g_use_per_node_hctx) { if (g_submit_queues != nr_online_nodes) { pr_warn("null_blk: submit_queues param is set to %u.\n", diff --git a/drivers/block/paride/pcd.c b/drivers/block/paride/pcd.c index a026211afb51..96670eefaeb2 100644 --- a/drivers/block/paride/pcd.c +++ b/drivers/block/paride/pcd.c @@ -137,7 +137,7 @@ enum {D_PRT, D_PRO, D_UNI, D_MOD, D_SLV, D_DLY}; #include <linux/delay.h> #include <linux/cdrom.h> #include <linux/spinlock.h> -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/mutex.h> #include <linux/uaccess.h> @@ -186,7 +186,8 @@ static int pcd_packet(struct cdrom_device_info *cdi, static int pcd_detect(void); static void pcd_probe_capabilities(void); static void do_pcd_read_drq(void); -static void do_pcd_request(struct request_queue * q); +static blk_status_t pcd_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd); static void do_pcd_read(void); struct pcd_unit { @@ -199,6 +200,8 @@ struct pcd_unit { char *name; /* pcd0, pcd1, etc */ struct cdrom_device_info info; /* uniform cdrom interface */ struct gendisk *disk; + struct blk_mq_tag_set tag_set; + struct list_head rq_list; }; static struct pcd_unit pcd[PCD_UNITS]; @@ -292,6 +295,10 @@ static const struct cdrom_device_ops pcd_dops = { CDC_CD_RW, }; +static const struct blk_mq_ops pcd_mq_ops = { + .queue_rq = pcd_queue_rq, +}; + static void pcd_init_units(void) { struct pcd_unit *cd; @@ -300,13 +307,19 @@ static void pcd_init_units(void) pcd_drive_count = 0; for (unit = 0, cd = pcd; unit < PCD_UNITS; unit++, cd++) { struct gendisk *disk = alloc_disk(1); + if (!disk) continue; - disk->queue = blk_init_queue(do_pcd_request, &pcd_lock); - if (!disk->queue) { - put_disk(disk); + + disk->queue = blk_mq_init_sq_queue(&cd->tag_set, &pcd_mq_ops, + 1, BLK_MQ_F_SHOULD_MERGE); + if (IS_ERR(disk->queue)) { + disk->queue = NULL; continue; } + + INIT_LIST_HEAD(&cd->rq_list); + disk->queue->queuedata = cd; blk_queue_bounce_limit(disk->queue, BLK_BOUNCE_HIGH); cd->disk = disk; cd->pi = &cd->pia; @@ -748,18 +761,18 @@ static int pcd_queue; static int set_next_request(void) { struct pcd_unit *cd; - struct request_queue *q; int old_pos = pcd_queue; do { cd = &pcd[pcd_queue]; - q = cd->present ? cd->disk->queue : NULL; if (++pcd_queue == PCD_UNITS) pcd_queue = 0; - if (q) { - pcd_req = blk_fetch_request(q); - if (pcd_req) - break; + if (cd->present && !list_empty(&cd->rq_list)) { + pcd_req = list_first_entry(&cd->rq_list, struct request, + queuelist); + list_del_init(&pcd_req->queuelist); + blk_mq_start_request(pcd_req); + break; } } while (pcd_queue != old_pos); @@ -768,33 +781,41 @@ static int set_next_request(void) static void pcd_request(void) { + struct pcd_unit *cd; + if (pcd_busy) return; - while (1) { - if (!pcd_req && !set_next_request()) - return; - if (rq_data_dir(pcd_req) == READ) { - struct pcd_unit *cd = pcd_req->rq_disk->private_data; - if (cd != pcd_current) - pcd_bufblk = -1; - pcd_current = cd; - pcd_sector = blk_rq_pos(pcd_req); - pcd_count = blk_rq_cur_sectors(pcd_req); - pcd_buf = bio_data(pcd_req->bio); - pcd_busy = 1; - ps_set_intr(do_pcd_read, NULL, 0, nice); - return; - } else { - __blk_end_request_all(pcd_req, BLK_STS_IOERR); - pcd_req = NULL; - } - } + if (!pcd_req && !set_next_request()) + return; + + cd = pcd_req->rq_disk->private_data; + if (cd != pcd_current) + pcd_bufblk = -1; + pcd_current = cd; + pcd_sector = blk_rq_pos(pcd_req); + pcd_count = blk_rq_cur_sectors(pcd_req); + pcd_buf = bio_data(pcd_req->bio); + pcd_busy = 1; + ps_set_intr(do_pcd_read, NULL, 0, nice); } -static void do_pcd_request(struct request_queue *q) +static blk_status_t pcd_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { + struct pcd_unit *cd = hctx->queue->queuedata; + + if (rq_data_dir(bd->rq) != READ) { + blk_mq_start_request(bd->rq); + return BLK_STS_IOERR; + } + + spin_lock_irq(&pcd_lock); + list_add_tail(&bd->rq->queuelist, &cd->rq_list); pcd_request(); + spin_unlock_irq(&pcd_lock); + + return BLK_STS_OK; } static inline void next_request(blk_status_t err) @@ -802,8 +823,10 @@ static inline void next_request(blk_status_t err) unsigned long saved_flags; spin_lock_irqsave(&pcd_lock, saved_flags); - if (!__blk_end_request_cur(pcd_req, err)) + if (!blk_update_request(pcd_req, err, blk_rq_cur_bytes(pcd_req))) { + __blk_mq_end_request(pcd_req, err); pcd_req = NULL; + } pcd_busy = 0; pcd_request(); spin_unlock_irqrestore(&pcd_lock, saved_flags); @@ -1011,6 +1034,7 @@ static void __exit pcd_exit(void) unregister_cdrom(&cd->info); } blk_cleanup_queue(cd->disk->queue); + blk_mq_free_tag_set(&cd->tag_set); put_disk(cd->disk); } unregister_blkdev(major, name); diff --git a/drivers/block/paride/pd.c b/drivers/block/paride/pd.c index 7cf947586fe4..ae4971e5d9a8 100644 --- a/drivers/block/paride/pd.c +++ b/drivers/block/paride/pd.c @@ -151,7 +151,7 @@ enum {D_PRT, D_PRO, D_UNI, D_MOD, D_GEO, D_SBY, D_DLY, D_SLV}; #include <linux/delay.h> #include <linux/hdreg.h> #include <linux/cdrom.h> /* for the eject ioctl */ -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/blkpg.h> #include <linux/kernel.h> #include <linux/mutex.h> @@ -236,6 +236,8 @@ struct pd_unit { int alt_geom; char name[PD_NAMELEN]; /* pda, pdb, etc ... */ struct gendisk *gd; + struct blk_mq_tag_set tag_set; + struct list_head rq_list; }; static struct pd_unit pd[PD_UNITS]; @@ -399,9 +401,17 @@ static int set_next_request(void) if (++pd_queue == PD_UNITS) pd_queue = 0; if (q) { - pd_req = blk_fetch_request(q); - if (pd_req) - break; + struct pd_unit *disk = q->queuedata; + + if (list_empty(&disk->rq_list)) + continue; + + pd_req = list_first_entry(&disk->rq_list, + struct request, + queuelist); + list_del_init(&pd_req->queuelist); + blk_mq_start_request(pd_req); + break; } } while (pd_queue != old_pos); @@ -412,7 +422,6 @@ static void run_fsm(void) { while (1) { enum action res; - unsigned long saved_flags; int stop = 0; if (!phase) { @@ -433,19 +442,24 @@ static void run_fsm(void) } switch(res = phase()) { - case Ok: case Fail: + case Ok: case Fail: { + blk_status_t err; + + err = res == Ok ? 0 : BLK_STS_IOERR; pi_disconnect(pi_current); pd_claimed = 0; phase = NULL; - spin_lock_irqsave(&pd_lock, saved_flags); - if (!__blk_end_request_cur(pd_req, - res == Ok ? 0 : BLK_STS_IOERR)) { - if (!set_next_request()) - stop = 1; + spin_lock_irq(&pd_lock); + if (!blk_update_request(pd_req, err, + blk_rq_cur_bytes(pd_req))) { + __blk_mq_end_request(pd_req, err); + pd_req = NULL; + stop = !set_next_request(); } - spin_unlock_irqrestore(&pd_lock, saved_flags); + spin_unlock_irq(&pd_lock); if (stop) return; + } /* fall through */ case Hold: schedule_fsm(); @@ -505,11 +519,17 @@ static int pd_next_buf(void) if (pd_count) return 0; spin_lock_irqsave(&pd_lock, saved_flags); - __blk_end_request_cur(pd_req, 0); - pd_count = blk_rq_cur_sectors(pd_req); - pd_buf = bio_data(pd_req->bio); + if (!blk_update_request(pd_req, 0, blk_rq_cur_bytes(pd_req))) { + __blk_mq_end_request(pd_req, 0); + pd_req = NULL; + pd_count = 0; + pd_buf = NULL; + } else { + pd_count = blk_rq_cur_sectors(pd_req); + pd_buf = bio_data(pd_req->bio); + } spin_unlock_irqrestore(&pd_lock, saved_flags); - return 0; + return !pd_count; } static unsigned long pd_timeout; @@ -726,15 +746,21 @@ static enum action pd_identify(struct pd_unit *disk) /* end of io request engine */ -static void do_pd_request(struct request_queue * q) +static blk_status_t pd_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { - if (pd_req) - return; - pd_req = blk_fetch_request(q); - if (!pd_req) - return; + struct pd_unit *disk = hctx->queue->queuedata; + + spin_lock_irq(&pd_lock); + if (!pd_req) { + pd_req = bd->rq; + blk_mq_start_request(pd_req); + } else + list_add_tail(&bd->rq->queuelist, &disk->rq_list); + spin_unlock_irq(&pd_lock); - schedule_fsm(); + run_fsm(); + return BLK_STS_OK; } static int pd_special_command(struct pd_unit *disk, @@ -847,23 +873,33 @@ static const struct block_device_operations pd_fops = { /* probing */ +static const struct blk_mq_ops pd_mq_ops = { + .queue_rq = pd_queue_rq, +}; + static void pd_probe_drive(struct pd_unit *disk) { - struct gendisk *p = alloc_disk(1 << PD_BITS); + struct gendisk *p; + + p = alloc_disk(1 << PD_BITS); if (!p) return; + strcpy(p->disk_name, disk->name); p->fops = &pd_fops; p->major = major; p->first_minor = (disk - pd) << PD_BITS; disk->gd = p; p->private_data = disk; - p->queue = blk_init_queue(do_pd_request, &pd_lock); - if (!p->queue) { - disk->gd = NULL; - put_disk(p); + + p->queue = blk_mq_init_sq_queue(&disk->tag_set, &pd_mq_ops, 2, + BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING); + if (IS_ERR(p->queue)) { + p->queue = NULL; return; } + + p->queue->queuedata = disk; blk_queue_max_hw_sectors(p->queue, cluster); blk_queue_bounce_limit(p->queue, BLK_BOUNCE_HIGH); @@ -895,6 +931,7 @@ static int pd_detect(void) disk->standby = parm[D_SBY]; if (parm[D_PRT]) pd_drive_count++; + INIT_LIST_HEAD(&disk->rq_list); } par_drv = pi_register_driver(name); @@ -972,6 +1009,7 @@ static void __exit pd_exit(void) disk->gd = NULL; del_gendisk(p); blk_cleanup_queue(p->queue); + blk_mq_free_tag_set(&disk->tag_set); put_disk(p); pi_release(disk->pi); } diff --git a/drivers/block/paride/pf.c b/drivers/block/paride/pf.c index eef7a91f667d..e92e7a8eeeb2 100644 --- a/drivers/block/paride/pf.c +++ b/drivers/block/paride/pf.c @@ -152,7 +152,7 @@ enum {D_PRT, D_PRO, D_UNI, D_MOD, D_SLV, D_LUN, D_DLY}; #include <linux/hdreg.h> #include <linux/cdrom.h> #include <linux/spinlock.h> -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/blkpg.h> #include <linux/mutex.h> #include <linux/uaccess.h> @@ -206,7 +206,8 @@ module_param_array(drive3, int, NULL, 0); #define ATAPI_WRITE_10 0x2a static int pf_open(struct block_device *bdev, fmode_t mode); -static void do_pf_request(struct request_queue * q); +static blk_status_t pf_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd); static int pf_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg); static int pf_getgeo(struct block_device *bdev, struct hd_geometry *geo); @@ -238,6 +239,8 @@ struct pf_unit { int present; /* device present ? */ char name[PF_NAMELEN]; /* pf0, pf1, ... */ struct gendisk *disk; + struct blk_mq_tag_set tag_set; + struct list_head rq_list; }; static struct pf_unit units[PF_UNITS]; @@ -277,6 +280,10 @@ static const struct block_device_operations pf_fops = { .check_events = pf_check_events, }; +static const struct blk_mq_ops pf_mq_ops = { + .queue_rq = pf_queue_rq, +}; + static void __init pf_init_units(void) { struct pf_unit *pf; @@ -284,14 +291,22 @@ static void __init pf_init_units(void) pf_drive_count = 0; for (unit = 0, pf = units; unit < PF_UNITS; unit++, pf++) { - struct gendisk *disk = alloc_disk(1); + struct gendisk *disk; + + disk = alloc_disk(1); if (!disk) continue; - disk->queue = blk_init_queue(do_pf_request, &pf_spin_lock); - if (!disk->queue) { + + disk->queue = blk_mq_init_sq_queue(&pf->tag_set, &pf_mq_ops, + 1, BLK_MQ_F_SHOULD_MERGE); + if (IS_ERR(disk->queue)) { put_disk(disk); - return; + disk->queue = NULL; + continue; } + + INIT_LIST_HEAD(&pf->rq_list); + disk->queue->queuedata = pf; blk_queue_max_segments(disk->queue, cluster); blk_queue_bounce_limit(disk->queue, BLK_BOUNCE_HIGH); pf->disk = disk; @@ -784,18 +799,18 @@ static int pf_queue; static int set_next_request(void) { struct pf_unit *pf; - struct request_queue *q; int old_pos = pf_queue; do { pf = &units[pf_queue]; - q = pf->present ? pf->disk->queue : NULL; if (++pf_queue == PF_UNITS) pf_queue = 0; - if (q) { - pf_req = blk_fetch_request(q); - if (pf_req) - break; + if (pf->present && !list_empty(&pf->rq_list)) { + pf_req = list_first_entry(&pf->rq_list, struct request, + queuelist); + list_del_init(&pf_req->queuelist); + blk_mq_start_request(pf_req); + break; } } while (pf_queue != old_pos); @@ -804,8 +819,12 @@ static int set_next_request(void) static void pf_end_request(blk_status_t err) { - if (pf_req && !__blk_end_request_cur(pf_req, err)) + if (!pf_req) + return; + if (!blk_update_request(pf_req, err, blk_rq_cur_bytes(pf_req))) { + __blk_mq_end_request(pf_req, err); pf_req = NULL; + } } static void pf_request(void) @@ -842,9 +861,17 @@ repeat: } } -static void do_pf_request(struct request_queue *q) +static blk_status_t pf_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { + struct pf_unit *pf = hctx->queue->queuedata; + + spin_lock_irq(&pf_spin_lock); + list_add_tail(&bd->rq->queuelist, &pf->rq_list); pf_request(); + spin_unlock_irq(&pf_spin_lock); + + return BLK_STS_OK; } static int pf_next_buf(void) @@ -1024,6 +1051,7 @@ static void __exit pf_exit(void) continue; del_gendisk(pf->disk); blk_cleanup_queue(pf->disk->queue); + blk_mq_free_tag_set(&pf->tag_set); put_disk(pf->disk); pi_release(pf->pi); } diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index 6f1d25c1eb64..9381f4e3b221 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -2645,7 +2645,7 @@ static int pkt_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, */ if (pd->refcnt == 1) pkt_lock_door(pd, 0); - /* fallthru */ + /* fall through */ /* * forward selected CDROM ioctls to CD-ROM, for UDF */ diff --git a/drivers/block/ps3disk.c b/drivers/block/ps3disk.c index afe1508d82c6..4e1d9b31f60c 100644 --- a/drivers/block/ps3disk.c +++ b/drivers/block/ps3disk.c @@ -19,7 +19,7 @@ */ #include <linux/ata.h> -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/slab.h> #include <linux/module.h> @@ -42,6 +42,7 @@ struct ps3disk_private { spinlock_t lock; /* Request queue spinlock */ struct request_queue *queue; + struct blk_mq_tag_set tag_set; struct gendisk *gendisk; unsigned int blocking_factor; struct request *req; @@ -118,8 +119,8 @@ static void ps3disk_scatter_gather(struct ps3_storage_device *dev, } } -static int ps3disk_submit_request_sg(struct ps3_storage_device *dev, - struct request *req) +static blk_status_t ps3disk_submit_request_sg(struct ps3_storage_device *dev, + struct request *req) { struct ps3disk_private *priv = ps3_system_bus_get_drvdata(&dev->sbd); int write = rq_data_dir(req), res; @@ -158,16 +159,15 @@ static int ps3disk_submit_request_sg(struct ps3_storage_device *dev, if (res) { dev_err(&dev->sbd.core, "%s:%u: %s failed %d\n", __func__, __LINE__, op, res); - __blk_end_request_all(req, BLK_STS_IOERR); - return 0; + return BLK_STS_IOERR; } priv->req = req; - return 1; + return BLK_STS_OK; } -static int ps3disk_submit_flush_request(struct ps3_storage_device *dev, - struct request *req) +static blk_status_t ps3disk_submit_flush_request(struct ps3_storage_device *dev, + struct request *req) { struct ps3disk_private *priv = ps3_system_bus_get_drvdata(&dev->sbd); u64 res; @@ -180,50 +180,45 @@ static int ps3disk_submit_flush_request(struct ps3_storage_device *dev, if (res) { dev_err(&dev->sbd.core, "%s:%u: sync cache failed 0x%llx\n", __func__, __LINE__, res); - __blk_end_request_all(req, BLK_STS_IOERR); - return 0; + return BLK_STS_IOERR; } priv->req = req; - return 1; + return BLK_STS_OK; } -static void ps3disk_do_request(struct ps3_storage_device *dev, - struct request_queue *q) +static blk_status_t ps3disk_do_request(struct ps3_storage_device *dev, + struct request *req) { - struct request *req; - dev_dbg(&dev->sbd.core, "%s:%u\n", __func__, __LINE__); - while ((req = blk_fetch_request(q))) { - switch (req_op(req)) { - case REQ_OP_FLUSH: - if (ps3disk_submit_flush_request(dev, req)) - return; - break; - case REQ_OP_READ: - case REQ_OP_WRITE: - if (ps3disk_submit_request_sg(dev, req)) - return; - break; - default: - blk_dump_rq_flags(req, DEVICE_NAME " bad request"); - __blk_end_request_all(req, BLK_STS_IOERR); - } + switch (req_op(req)) { + case REQ_OP_FLUSH: + return ps3disk_submit_flush_request(dev, req); + case REQ_OP_READ: + case REQ_OP_WRITE: + return ps3disk_submit_request_sg(dev, req); + default: + blk_dump_rq_flags(req, DEVICE_NAME " bad request"); + return BLK_STS_IOERR; } } -static void ps3disk_request(struct request_queue *q) +static blk_status_t ps3disk_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { + struct request_queue *q = hctx->queue; struct ps3_storage_device *dev = q->queuedata; struct ps3disk_private *priv = ps3_system_bus_get_drvdata(&dev->sbd); + blk_status_t ret; - if (priv->req) { - dev_dbg(&dev->sbd.core, "%s:%u busy\n", __func__, __LINE__); - return; - } + blk_mq_start_request(bd->rq); + + spin_lock_irq(&priv->lock); + ret = ps3disk_do_request(dev, bd->rq); + spin_unlock_irq(&priv->lock); - ps3disk_do_request(dev, q); + return ret; } static irqreturn_t ps3disk_interrupt(int irq, void *data) @@ -280,11 +275,11 @@ static irqreturn_t ps3disk_interrupt(int irq, void *data) } spin_lock(&priv->lock); - __blk_end_request_all(req, error); priv->req = NULL; - ps3disk_do_request(dev, priv->queue); + blk_mq_end_request(req, error); spin_unlock(&priv->lock); + blk_mq_run_hw_queues(priv->queue, true); return IRQ_HANDLED; } @@ -404,6 +399,10 @@ static unsigned long ps3disk_mask; static DEFINE_MUTEX(ps3disk_mask_mutex); +static const struct blk_mq_ops ps3disk_mq_ops = { + .queue_rq = ps3disk_queue_rq, +}; + static int ps3disk_probe(struct ps3_system_bus_device *_dev) { struct ps3_storage_device *dev = to_ps3_storage_device(&_dev->core); @@ -454,11 +453,12 @@ static int ps3disk_probe(struct ps3_system_bus_device *_dev) ps3disk_identify(dev); - queue = blk_init_queue(ps3disk_request, &priv->lock); - if (!queue) { - dev_err(&dev->sbd.core, "%s:%u: blk_init_queue failed\n", + queue = blk_mq_init_sq_queue(&priv->tag_set, &ps3disk_mq_ops, 1, + BLK_MQ_F_SHOULD_MERGE); + if (IS_ERR(queue)) { + dev_err(&dev->sbd.core, "%s:%u: blk_mq_init_queue failed\n", __func__, __LINE__); - error = -ENOMEM; + error = PTR_ERR(queue); goto fail_teardown; } @@ -500,11 +500,12 @@ static int ps3disk_probe(struct ps3_system_bus_device *_dev) gendisk->disk_name, priv->model, priv->raw_capacity >> 11, get_capacity(gendisk) >> 11); - device_add_disk(&dev->sbd.core, gendisk); + device_add_disk(&dev->sbd.core, gendisk, NULL); return 0; fail_cleanup_queue: blk_cleanup_queue(queue); + blk_mq_free_tag_set(&priv->tag_set); fail_teardown: ps3stor_teardown(dev); fail_free_bounce: @@ -530,6 +531,7 @@ static int ps3disk_remove(struct ps3_system_bus_device *_dev) mutex_unlock(&ps3disk_mask_mutex); del_gendisk(priv->gendisk); blk_cleanup_queue(priv->queue); + blk_mq_free_tag_set(&priv->tag_set); put_disk(priv->gendisk); dev_notice(&dev->sbd.core, "Synchronizing disk cache\n"); ps3disk_sync_cache(dev); diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c index 1e3d5de9d838..c0c50816a10b 100644 --- a/drivers/block/ps3vram.c +++ b/drivers/block/ps3vram.c @@ -769,7 +769,7 @@ static int ps3vram_probe(struct ps3_system_bus_device *dev) dev_info(&dev->core, "%s: Using %lu MiB of GPU memory\n", gendisk->disk_name, get_capacity(gendisk) >> 11); - device_add_disk(&dev->core, gendisk); + device_add_disk(&dev->core, gendisk, NULL); return 0; fail_cleanup_queue: diff --git a/drivers/block/rsxx/core.c b/drivers/block/rsxx/core.c index f2c631ce793c..639051502181 100644 --- a/drivers/block/rsxx/core.c +++ b/drivers/block/rsxx/core.c @@ -782,7 +782,7 @@ static int rsxx_pci_probe(struct pci_dev *dev, pci_set_master(dev); pci_set_dma_max_seg_size(dev, RSXX_HW_BLK_SIZE); - st = pci_set_dma_mask(dev, DMA_BIT_MASK(64)); + st = dma_set_mask(&dev->dev, DMA_BIT_MASK(64)); if (st) { dev_err(CARD_TO_DEV(card), "No usable DMA configuration,aborting\n"); diff --git a/drivers/block/rsxx/cregs.c b/drivers/block/rsxx/cregs.c index c148e83e4ed7..d9a8758682c9 100644 --- a/drivers/block/rsxx/cregs.c +++ b/drivers/block/rsxx/cregs.c @@ -276,7 +276,7 @@ static void creg_cmd_done(struct work_struct *work) st = -EIO; } - if ((cmd->op == CREG_OP_READ)) { + if (cmd->op == CREG_OP_READ) { unsigned int cnt8 = ioread32(card->regmap + CREG_CNT); /* Paranoid Sanity Checks */ diff --git a/drivers/block/rsxx/dev.c b/drivers/block/rsxx/dev.c index 1a92f9e65937..3894aa0f350b 100644 --- a/drivers/block/rsxx/dev.c +++ b/drivers/block/rsxx/dev.c @@ -226,7 +226,7 @@ int rsxx_attach_dev(struct rsxx_cardinfo *card) set_capacity(card->gendisk, card->size8 >> 9); else set_capacity(card->gendisk, 0); - device_add_disk(CARD_TO_DEV(card), card->gendisk); + device_add_disk(CARD_TO_DEV(card), card->gendisk, NULL); card->bdev_attached = 1; } diff --git a/drivers/block/rsxx/dma.c b/drivers/block/rsxx/dma.c index 8fbc1bf6db3d..af9cf0215164 100644 --- a/drivers/block/rsxx/dma.c +++ b/drivers/block/rsxx/dma.c @@ -224,12 +224,12 @@ static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card) static void rsxx_free_dma(struct rsxx_dma_ctrl *ctrl, struct rsxx_dma *dma) { if (dma->cmd != HW_CMD_BLK_DISCARD) { - if (!pci_dma_mapping_error(ctrl->card->dev, dma->dma_addr)) { - pci_unmap_page(ctrl->card->dev, dma->dma_addr, + if (!dma_mapping_error(&ctrl->card->dev->dev, dma->dma_addr)) { + dma_unmap_page(&ctrl->card->dev->dev, dma->dma_addr, get_dma_size(dma), dma->cmd == HW_CMD_BLK_WRITE ? - PCI_DMA_TODEVICE : - PCI_DMA_FROMDEVICE); + DMA_TO_DEVICE : + DMA_FROM_DEVICE); } } @@ -438,23 +438,23 @@ static void rsxx_issue_dmas(struct rsxx_dma_ctrl *ctrl) if (dma->cmd != HW_CMD_BLK_DISCARD) { if (dma->cmd == HW_CMD_BLK_WRITE) - dir = PCI_DMA_TODEVICE; + dir = DMA_TO_DEVICE; else - dir = PCI_DMA_FROMDEVICE; + dir = DMA_FROM_DEVICE; /* - * The function pci_map_page is placed here because we + * The function dma_map_page is placed here because we * can only, by design, issue up to 255 commands to the * hardware at one time per DMA channel. So the maximum * amount of mapped memory would be 255 * 4 channels * * 4096 Bytes which is less than 2GB, the limit of a x8 - * Non-HWWD PCIe slot. This way the pci_map_page + * Non-HWWD PCIe slot. This way the dma_map_page * function should never fail because of a lack of * mappable memory. */ - dma->dma_addr = pci_map_page(ctrl->card->dev, dma->page, + dma->dma_addr = dma_map_page(&ctrl->card->dev->dev, dma->page, dma->pg_off, dma->sub_page.cnt << 9, dir); - if (pci_dma_mapping_error(ctrl->card->dev, dma->dma_addr)) { + if (dma_mapping_error(&ctrl->card->dev->dev, dma->dma_addr)) { push_tracker(ctrl->trackers, tag); rsxx_complete_dma(ctrl, dma, DMA_CANCELLED); continue; @@ -776,10 +776,10 @@ bvec_err: /*----------------- DMA Engine Initialization & Setup -------------------*/ int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl) { - ctrl->status.buf = pci_alloc_consistent(dev, STATUS_BUFFER_SIZE8, - &ctrl->status.dma_addr); - ctrl->cmd.buf = pci_alloc_consistent(dev, COMMAND_BUFFER_SIZE8, - &ctrl->cmd.dma_addr); + ctrl->status.buf = dma_alloc_coherent(&dev->dev, STATUS_BUFFER_SIZE8, + &ctrl->status.dma_addr, GFP_KERNEL); + ctrl->cmd.buf = dma_alloc_coherent(&dev->dev, COMMAND_BUFFER_SIZE8, + &ctrl->cmd.dma_addr, GFP_KERNEL); if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL) return -ENOMEM; @@ -962,12 +962,12 @@ failed_dma_setup: vfree(ctrl->trackers); if (ctrl->status.buf) - pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8, - ctrl->status.buf, - ctrl->status.dma_addr); + dma_free_coherent(&card->dev->dev, STATUS_BUFFER_SIZE8, + ctrl->status.buf, + ctrl->status.dma_addr); if (ctrl->cmd.buf) - pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8, - ctrl->cmd.buf, ctrl->cmd.dma_addr); + dma_free_coherent(&card->dev->dev, COMMAND_BUFFER_SIZE8, + ctrl->cmd.buf, ctrl->cmd.dma_addr); } return st; @@ -1023,10 +1023,10 @@ void rsxx_dma_destroy(struct rsxx_cardinfo *card) vfree(ctrl->trackers); - pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8, - ctrl->status.buf, ctrl->status.dma_addr); - pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8, - ctrl->cmd.buf, ctrl->cmd.dma_addr); + dma_free_coherent(&card->dev->dev, STATUS_BUFFER_SIZE8, + ctrl->status.buf, ctrl->status.dma_addr); + dma_free_coherent(&card->dev->dev, COMMAND_BUFFER_SIZE8, + ctrl->cmd.buf, ctrl->cmd.dma_addr); } } @@ -1059,11 +1059,11 @@ int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card) card->ctrl[i].stats.reads_issued--; if (dma->cmd != HW_CMD_BLK_DISCARD) { - pci_unmap_page(card->dev, dma->dma_addr, + dma_unmap_page(&card->dev->dev, dma->dma_addr, get_dma_size(dma), dma->cmd == HW_CMD_BLK_WRITE ? - PCI_DMA_TODEVICE : - PCI_DMA_FROMDEVICE); + DMA_TO_DEVICE : + DMA_FROM_DEVICE); } list_add_tail(&dma->list, &issued_dmas[i]); diff --git a/drivers/block/skd_main.c b/drivers/block/skd_main.c index 87b9e7fbf062..7c5fc6942f32 100644 --- a/drivers/block/skd_main.c +++ b/drivers/block/skd_main.c @@ -632,7 +632,7 @@ static bool skd_preop_sg_list(struct skd_device *skdev, * Map scatterlist to PCI bus addresses. * Note PCI might change the number of entries. */ - n_sg = pci_map_sg(skdev->pdev, sgl, n_sg, skreq->data_dir); + n_sg = dma_map_sg(&skdev->pdev->dev, sgl, n_sg, skreq->data_dir); if (n_sg <= 0) return false; @@ -682,7 +682,8 @@ static void skd_postop_sg_list(struct skd_device *skdev, skreq->sksg_list[skreq->n_sg - 1].next_desc_ptr = skreq->sksg_dma_address + ((skreq->n_sg) * sizeof(struct fit_sg_descriptor)); - pci_unmap_sg(skdev->pdev, &skreq->sg[0], skreq->n_sg, skreq->data_dir); + dma_unmap_sg(&skdev->pdev->dev, &skreq->sg[0], skreq->n_sg, + skreq->data_dir); } /* @@ -1416,7 +1417,7 @@ static void skd_resolve_req_exception(struct skd_device *skdev, case SKD_CHECK_STATUS_BUSY_IMMINENT: skd_log_skreq(skdev, skreq, "retry(busy)"); - blk_requeue_request(skdev->queue, req); + blk_mq_requeue_request(req, true); dev_info(&skdev->pdev->dev, "drive BUSY imminent\n"); skdev->state = SKD_DRVR_STATE_BUSY_IMMINENT; skdev->timer_countdown = SKD_TIMER_MINUTES(20); @@ -1426,7 +1427,7 @@ static void skd_resolve_req_exception(struct skd_device *skdev, case SKD_CHECK_STATUS_REQUEUE_REQUEST: if ((unsigned long) ++req->special < SKD_MAX_RETRIES) { skd_log_skreq(skdev, skreq, "retry"); - blk_requeue_request(skdev->queue, req); + blk_mq_requeue_request(req, true); break; } /* fall through */ @@ -2632,8 +2633,8 @@ static int skd_cons_skcomp(struct skd_device *skdev) "comp pci_alloc, total bytes %zd entries %d\n", SKD_SKCOMP_SIZE, SKD_N_COMPLETION_ENTRY); - skcomp = pci_zalloc_consistent(skdev->pdev, SKD_SKCOMP_SIZE, - &skdev->cq_dma_address); + skcomp = dma_zalloc_coherent(&skdev->pdev->dev, SKD_SKCOMP_SIZE, + &skdev->cq_dma_address, GFP_KERNEL); if (skcomp == NULL) { rc = -ENOMEM; @@ -2674,10 +2675,10 @@ static int skd_cons_skmsg(struct skd_device *skdev) skmsg->id = i + SKD_ID_FIT_MSG; - skmsg->msg_buf = pci_alloc_consistent(skdev->pdev, - SKD_N_FITMSG_BYTES, - &skmsg->mb_dma_address); - + skmsg->msg_buf = dma_alloc_coherent(&skdev->pdev->dev, + SKD_N_FITMSG_BYTES, + &skmsg->mb_dma_address, + GFP_KERNEL); if (skmsg->msg_buf == NULL) { rc = -ENOMEM; goto err_out; @@ -2971,8 +2972,8 @@ err_out: static void skd_free_skcomp(struct skd_device *skdev) { if (skdev->skcomp_table) - pci_free_consistent(skdev->pdev, SKD_SKCOMP_SIZE, - skdev->skcomp_table, skdev->cq_dma_address); + dma_free_coherent(&skdev->pdev->dev, SKD_SKCOMP_SIZE, + skdev->skcomp_table, skdev->cq_dma_address); skdev->skcomp_table = NULL; skdev->cq_dma_address = 0; @@ -2991,8 +2992,8 @@ static void skd_free_skmsg(struct skd_device *skdev) skmsg = &skdev->skmsg_table[i]; if (skmsg->msg_buf != NULL) { - pci_free_consistent(skdev->pdev, SKD_N_FITMSG_BYTES, - skmsg->msg_buf, + dma_free_coherent(&skdev->pdev->dev, SKD_N_FITMSG_BYTES, + skmsg->msg_buf, skmsg->mb_dma_address); } skmsg->msg_buf = NULL; @@ -3104,7 +3105,7 @@ static int skd_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo) static int skd_bdev_attach(struct device *parent, struct skd_device *skdev) { dev_dbg(&skdev->pdev->dev, "add_disk\n"); - device_add_disk(parent, skdev->disk); + device_add_disk(parent, skdev->disk, NULL); return 0; } @@ -3172,18 +3173,12 @@ static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) rc = pci_request_regions(pdev, DRV_NAME); if (rc) goto err_out; - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); - if (!rc) { - if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) { - dev_err(&pdev->dev, "consistent DMA mask error %d\n", - rc); - } - } else { - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); - if (rc) { - dev_err(&pdev->dev, "DMA mask error %d\n", rc); - goto err_out_regions; - } + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (rc) + dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (rc) { + dev_err(&pdev->dev, "DMA mask error %d\n", rc); + goto err_out_regions; } if (!skd_major) { @@ -3367,20 +3362,12 @@ static int skd_pci_resume(struct pci_dev *pdev) rc = pci_request_regions(pdev, DRV_NAME); if (rc) goto err_out; - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); - if (!rc) { - if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) { - - dev_err(&pdev->dev, "consistent DMA mask error %d\n", - rc); - } - } else { - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); - if (rc) { - - dev_err(&pdev->dev, "DMA mask error %d\n", rc); - goto err_out_regions; - } + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (rc) + dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (rc) { + dev_err(&pdev->dev, "DMA mask error %d\n", rc); + goto err_out_regions; } pci_set_master(pdev); diff --git a/drivers/block/sunvdc.c b/drivers/block/sunvdc.c index 5ca56bfae63c..b54fa6726303 100644 --- a/drivers/block/sunvdc.c +++ b/drivers/block/sunvdc.c @@ -36,6 +36,10 @@ MODULE_VERSION(DRV_MODULE_VERSION); #define VDC_TX_RING_SIZE 512 #define VDC_DEFAULT_BLK_SIZE 512 +#define MAX_XFER_BLKS (128 * 1024) +#define MAX_XFER_SIZE (MAX_XFER_BLKS / VDC_DEFAULT_BLK_SIZE) +#define MAX_RING_COOKIES ((MAX_XFER_BLKS / PAGE_SIZE) + 2) + #define WAITING_FOR_LINK_UP 0x01 #define WAITING_FOR_TX_SPACE 0x02 #define WAITING_FOR_GEN_CMD 0x04 @@ -450,7 +454,7 @@ static int __send_request(struct request *req) { struct vdc_port *port = req->rq_disk->private_data; struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; - struct scatterlist sg[port->ring_cookies]; + struct scatterlist sg[MAX_RING_COOKIES]; struct vdc_req_entry *rqe; struct vio_disk_desc *desc; unsigned int map_perm; @@ -458,6 +462,9 @@ static int __send_request(struct request *req) u64 len; u8 op; + if (WARN_ON(port->ring_cookies > MAX_RING_COOKIES)) + return -EINVAL; + map_perm = LDC_MAP_SHADOW | LDC_MAP_DIRECT | LDC_MAP_IO; if (rq_data_dir(req) == READ) { @@ -850,7 +857,7 @@ static int probe_disk(struct vdc_port *port) port->vdisk_size, (port->vdisk_size >> (20 - 9)), port->vio.ver.major, port->vio.ver.minor); - device_add_disk(&port->vio.vdev->dev, g); + device_add_disk(&port->vio.vdev->dev, g, NULL); return 0; } @@ -984,9 +991,8 @@ static int vdc_port_probe(struct vio_dev *vdev, const struct vio_device_id *id) goto err_out_free_port; port->vdisk_block_size = VDC_DEFAULT_BLK_SIZE; - port->max_xfer_size = ((128 * 1024) / port->vdisk_block_size); - port->ring_cookies = ((port->max_xfer_size * - port->vdisk_block_size) / PAGE_SIZE) + 2; + port->max_xfer_size = MAX_XFER_SIZE; + port->ring_cookies = MAX_RING_COOKIES; err = vio_ldc_alloc(&port->vio, &vdc_ldc_cfg, port); if (err) diff --git a/drivers/block/swim.c b/drivers/block/swim.c index 0e31884a9519..3fa6fcc34790 100644 --- a/drivers/block/swim.c +++ b/drivers/block/swim.c @@ -19,7 +19,7 @@ #include <linux/module.h> #include <linux/fd.h> #include <linux/slab.h> -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/mutex.h> #include <linux/hdreg.h> #include <linux/kernel.h> @@ -190,6 +190,7 @@ struct floppy_state { int ref_count; struct gendisk *disk; + struct blk_mq_tag_set tag_set; /* parent controller */ @@ -211,7 +212,6 @@ enum head { struct swim_priv { struct swim __iomem *base; spinlock_t lock; - int fdc_queue; int floppy_count; struct floppy_state unit[FD_MAX_UNIT]; }; @@ -525,58 +525,36 @@ static blk_status_t floppy_read_sectors(struct floppy_state *fs, return 0; } -static struct request *swim_next_request(struct swim_priv *swd) +static blk_status_t swim_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { - struct request_queue *q; - struct request *rq; - int old_pos = swd->fdc_queue; + struct floppy_state *fs = hctx->queue->queuedata; + struct swim_priv *swd = fs->swd; + struct request *req = bd->rq; + blk_status_t err; - do { - q = swd->unit[swd->fdc_queue].disk->queue; - if (++swd->fdc_queue == swd->floppy_count) - swd->fdc_queue = 0; - if (q) { - rq = blk_fetch_request(q); - if (rq) - return rq; - } - } while (swd->fdc_queue != old_pos); + if (!spin_trylock_irq(&swd->lock)) + return BLK_STS_DEV_RESOURCE; - return NULL; -} + blk_mq_start_request(req); -static void do_fd_request(struct request_queue *q) -{ - struct swim_priv *swd = q->queuedata; - struct request *req; - struct floppy_state *fs; + if (!fs->disk_in || rq_data_dir(req) == WRITE) { + err = BLK_STS_IOERR; + goto out; + } - req = swim_next_request(swd); - while (req) { - blk_status_t err = BLK_STS_IOERR; + do { + err = floppy_read_sectors(fs, blk_rq_pos(req), + blk_rq_cur_sectors(req), + bio_data(req->bio)); + } while (blk_update_request(req, err, blk_rq_cur_bytes(req))); + __blk_mq_end_request(req, err); - fs = req->rq_disk->private_data; - if (blk_rq_pos(req) >= fs->total_secs) - goto done; - if (!fs->disk_in) - goto done; - if (rq_data_dir(req) == WRITE && fs->write_protected) - goto done; + err = BLK_STS_OK; +out: + spin_unlock_irq(&swd->lock); + return err; - switch (rq_data_dir(req)) { - case WRITE: - /* NOT IMPLEMENTED */ - break; - case READ: - err = floppy_read_sectors(fs, blk_rq_pos(req), - blk_rq_cur_sectors(req), - bio_data(req->bio)); - break; - } - done: - if (!__blk_end_request_cur(req, err)) - req = swim_next_request(swd); - } } static struct floppy_struct floppy_type[4] = { @@ -823,6 +801,10 @@ static int swim_add_floppy(struct swim_priv *swd, enum drive_location location) return 0; } +static const struct blk_mq_ops swim_mq_ops = { + .queue_rq = swim_queue_rq, +}; + static int swim_floppy_init(struct swim_priv *swd) { int err; @@ -852,20 +834,25 @@ static int swim_floppy_init(struct swim_priv *swd) spin_lock_init(&swd->lock); for (drive = 0; drive < swd->floppy_count; drive++) { + struct request_queue *q; + swd->unit[drive].disk = alloc_disk(1); if (swd->unit[drive].disk == NULL) { err = -ENOMEM; goto exit_put_disks; } - swd->unit[drive].disk->queue = blk_init_queue(do_fd_request, - &swd->lock); - if (!swd->unit[drive].disk->queue) { - err = -ENOMEM; + + q = blk_mq_init_sq_queue(&swd->unit[drive].tag_set, &swim_mq_ops, + 2, BLK_MQ_F_SHOULD_MERGE); + if (IS_ERR(q)) { + err = PTR_ERR(q); goto exit_put_disks; } + + swd->unit[drive].disk->queue = q; blk_queue_bounce_limit(swd->unit[drive].disk->queue, BLK_BOUNCE_HIGH); - swd->unit[drive].disk->queue->queuedata = swd; + swd->unit[drive].disk->queue->queuedata = &swd->unit[drive]; swd->unit[drive].swd = swd; } @@ -887,8 +874,18 @@ static int swim_floppy_init(struct swim_priv *swd) exit_put_disks: unregister_blkdev(FLOPPY_MAJOR, "fd"); - while (drive--) - put_disk(swd->unit[drive].disk); + do { + struct gendisk *disk = swd->unit[drive].disk; + + if (disk) { + if (disk->queue) { + blk_cleanup_queue(disk->queue); + disk->queue = NULL; + } + blk_mq_free_tag_set(&swd->unit[drive].tag_set); + put_disk(disk); + } + } while (drive--); return err; } @@ -961,6 +958,7 @@ static int swim_remove(struct platform_device *dev) for (drive = 0; drive < swd->floppy_count; drive++) { del_gendisk(swd->unit[drive].disk); blk_cleanup_queue(swd->unit[drive].disk->queue); + blk_mq_free_tag_set(&swd->unit[drive].tag_set); put_disk(swd->unit[drive].disk); } diff --git a/drivers/block/swim3.c b/drivers/block/swim3.c index 469541c1e51e..c1c676a33e4a 100644 --- a/drivers/block/swim3.c +++ b/drivers/block/swim3.c @@ -25,7 +25,7 @@ #include <linux/delay.h> #include <linux/fd.h> #include <linux/ioctl.h> -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/interrupt.h> #include <linux/mutex.h> #include <linux/module.h> @@ -206,6 +206,7 @@ struct floppy_state { char dbdma_cmd_space[5 * sizeof(struct dbdma_cmd)]; int index; struct request *cur_req; + struct blk_mq_tag_set tag_set; }; #define swim3_err(fmt, arg...) dev_err(&fs->mdev->ofdev.dev, "[fd%d] " fmt, fs->index, arg) @@ -260,16 +261,15 @@ static int floppy_revalidate(struct gendisk *disk); static bool swim3_end_request(struct floppy_state *fs, blk_status_t err, unsigned int nr_bytes) { struct request *req = fs->cur_req; - int rc; swim3_dbg(" end request, err=%d nr_bytes=%d, cur_req=%p\n", err, nr_bytes, req); if (err) nr_bytes = blk_rq_cur_bytes(req); - rc = __blk_end_request(req, err, nr_bytes); - if (rc) + if (blk_update_request(req, err, nr_bytes)) return true; + __blk_mq_end_request(req, err); fs->cur_req = NULL; return false; } @@ -309,86 +309,58 @@ static int swim3_readbit(struct floppy_state *fs, int bit) return (stat & DATA) == 0; } -static void start_request(struct floppy_state *fs) +static blk_status_t swim3_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { - struct request *req; + struct floppy_state *fs = hctx->queue->queuedata; + struct request *req = bd->rq; unsigned long x; - swim3_dbg("start request, initial state=%d\n", fs->state); - - if (fs->state == idle && fs->wanted) { - fs->state = available; - wake_up(&fs->wait); - return; + spin_lock_irq(&swim3_lock); + if (fs->cur_req || fs->state != idle) { + spin_unlock_irq(&swim3_lock); + return BLK_STS_DEV_RESOURCE; } - while (fs->state == idle) { - swim3_dbg("start request, idle loop, cur_req=%p\n", fs->cur_req); - if (!fs->cur_req) { - fs->cur_req = blk_fetch_request(disks[fs->index]->queue); - swim3_dbg(" fetched request %p\n", fs->cur_req); - if (!fs->cur_req) - break; - } - req = fs->cur_req; - - if (fs->mdev->media_bay && - check_media_bay(fs->mdev->media_bay) != MB_FD) { - swim3_dbg("%s", " media bay absent, dropping req\n"); - swim3_end_request(fs, BLK_STS_IOERR, 0); - continue; - } - -#if 0 /* This is really too verbose */ - swim3_dbg("do_fd_req: dev=%s cmd=%d sec=%ld nr_sec=%u buf=%p\n", - req->rq_disk->disk_name, req->cmd, - (long)blk_rq_pos(req), blk_rq_sectors(req), - bio_data(req->bio)); - swim3_dbg(" current_nr_sectors=%u\n", - blk_rq_cur_sectors(req)); -#endif - - if (blk_rq_pos(req) >= fs->total_secs) { - swim3_dbg(" pos out of bounds (%ld, max is %ld)\n", - (long)blk_rq_pos(req), (long)fs->total_secs); - swim3_end_request(fs, BLK_STS_IOERR, 0); - continue; - } - if (fs->ejected) { - swim3_dbg("%s", " disk ejected\n"); + blk_mq_start_request(req); + fs->cur_req = req; + if (fs->mdev->media_bay && + check_media_bay(fs->mdev->media_bay) != MB_FD) { + swim3_dbg("%s", " media bay absent, dropping req\n"); + swim3_end_request(fs, BLK_STS_IOERR, 0); + goto out; + } + if (fs->ejected) { + swim3_dbg("%s", " disk ejected\n"); + swim3_end_request(fs, BLK_STS_IOERR, 0); + goto out; + } + if (rq_data_dir(req) == WRITE) { + if (fs->write_prot < 0) + fs->write_prot = swim3_readbit(fs, WRITE_PROT); + if (fs->write_prot) { + swim3_dbg("%s", " try to write, disk write protected\n"); swim3_end_request(fs, BLK_STS_IOERR, 0); - continue; + goto out; } - - if (rq_data_dir(req) == WRITE) { - if (fs->write_prot < 0) - fs->write_prot = swim3_readbit(fs, WRITE_PROT); - if (fs->write_prot) { - swim3_dbg("%s", " try to write, disk write protected\n"); - swim3_end_request(fs, BLK_STS_IOERR, 0); - continue; - } - } - - /* Do not remove the cast. blk_rq_pos(req) is now a - * sector_t and can be 64 bits, but it will never go - * past 32 bits for this driver anyway, so we can - * safely cast it down and not have to do a 64/32 - * division - */ - fs->req_cyl = ((long)blk_rq_pos(req)) / fs->secpercyl; - x = ((long)blk_rq_pos(req)) % fs->secpercyl; - fs->head = x / fs->secpertrack; - fs->req_sector = x % fs->secpertrack + 1; - fs->state = do_transfer; - fs->retries = 0; - - act(fs); } -} -static void do_fd_request(struct request_queue * q) -{ - start_request(q->queuedata); + /* + * Do not remove the cast. blk_rq_pos(req) is now a sector_t and can be + * 64 bits, but it will never go past 32 bits for this driver anyway, so + * we can safely cast it down and not have to do a 64/32 division + */ + fs->req_cyl = ((long)blk_rq_pos(req)) / fs->secpercyl; + x = ((long)blk_rq_pos(req)) % fs->secpercyl; + fs->head = x / fs->secpertrack; + fs->req_sector = x % fs->secpertrack + 1; + fs->state = do_transfer; + fs->retries = 0; + + act(fs); + +out: + spin_unlock_irq(&swim3_lock); + return BLK_STS_OK; } static void set_timeout(struct floppy_state *fs, int nticks, @@ -585,7 +557,6 @@ static void scan_timeout(struct timer_list *t) if (fs->retries > 5) { swim3_end_request(fs, BLK_STS_IOERR, 0); fs->state = idle; - start_request(fs); } else { fs->state = jogging; act(fs); @@ -609,7 +580,6 @@ static void seek_timeout(struct timer_list *t) swim3_err("%s", "Seek timeout\n"); swim3_end_request(fs, BLK_STS_IOERR, 0); fs->state = idle; - start_request(fs); spin_unlock_irqrestore(&swim3_lock, flags); } @@ -638,7 +608,6 @@ static void settle_timeout(struct timer_list *t) swim3_err("%s", "Seek settle timeout\n"); swim3_end_request(fs, BLK_STS_IOERR, 0); fs->state = idle; - start_request(fs); unlock: spin_unlock_irqrestore(&swim3_lock, flags); } @@ -667,7 +636,6 @@ static void xfer_timeout(struct timer_list *t) (long)blk_rq_pos(fs->cur_req)); swim3_end_request(fs, BLK_STS_IOERR, 0); fs->state = idle; - start_request(fs); spin_unlock_irqrestore(&swim3_lock, flags); } @@ -704,7 +672,6 @@ static irqreturn_t swim3_interrupt(int irq, void *dev_id) if (fs->retries > 5) { swim3_end_request(fs, BLK_STS_IOERR, 0); fs->state = idle; - start_request(fs); } else { fs->state = jogging; act(fs); @@ -796,7 +763,6 @@ static irqreturn_t swim3_interrupt(int irq, void *dev_id) fs->state, rq_data_dir(req), intr, err); swim3_end_request(fs, BLK_STS_IOERR, 0); fs->state = idle; - start_request(fs); break; } fs->retries = 0; @@ -813,8 +779,6 @@ static irqreturn_t swim3_interrupt(int irq, void *dev_id) } else fs->state = idle; } - if (fs->state == idle) - start_request(fs); break; default: swim3_err("Don't know what to do in state %d\n", fs->state); @@ -862,14 +826,19 @@ static int grab_drive(struct floppy_state *fs, enum swim_state state, static void release_drive(struct floppy_state *fs) { + struct request_queue *q = disks[fs->index]->queue; unsigned long flags; swim3_dbg("%s", "-> release drive\n"); spin_lock_irqsave(&swim3_lock, flags); fs->state = idle; - start_request(fs); spin_unlock_irqrestore(&swim3_lock, flags); + + blk_mq_freeze_queue(q); + blk_mq_quiesce_queue(q); + blk_mq_unquiesce_queue(q); + blk_mq_unfreeze_queue(q); } static int fd_eject(struct floppy_state *fs) @@ -1089,6 +1058,10 @@ static const struct block_device_operations floppy_fops = { .revalidate_disk= floppy_revalidate, }; +static const struct blk_mq_ops swim3_mq_ops = { + .queue_rq = swim3_queue_rq, +}; + static void swim3_mb_event(struct macio_dev* mdev, int mb_state) { struct floppy_state *fs = macio_get_drvdata(mdev); @@ -1202,47 +1175,63 @@ static int swim3_add_device(struct macio_dev *mdev, int index) static int swim3_attach(struct macio_dev *mdev, const struct of_device_id *match) { + struct floppy_state *fs; struct gendisk *disk; - int index, rc; + int rc; - index = floppy_count++; - if (index >= MAX_FLOPPIES) + if (floppy_count >= MAX_FLOPPIES) return -ENXIO; - /* Add the drive */ - rc = swim3_add_device(mdev, index); - if (rc) - return rc; - /* Now register that disk. Same comment about failure handling */ - disk = disks[index] = alloc_disk(1); - if (disk == NULL) - return -ENOMEM; - disk->queue = blk_init_queue(do_fd_request, &swim3_lock); - if (disk->queue == NULL) { - put_disk(disk); - return -ENOMEM; + if (floppy_count == 0) { + rc = register_blkdev(FLOPPY_MAJOR, "fd"); + if (rc) + return rc; } - blk_queue_bounce_limit(disk->queue, BLK_BOUNCE_HIGH); - disk->queue->queuedata = &floppy_states[index]; - if (index == 0) { - /* If we failed, there isn't much we can do as the driver is still - * too dumb to remove the device, just bail out - */ - if (register_blkdev(FLOPPY_MAJOR, "fd")) - return 0; + fs = &floppy_states[floppy_count]; + + disk = alloc_disk(1); + if (disk == NULL) { + rc = -ENOMEM; + goto out_unregister; + } + + disk->queue = blk_mq_init_sq_queue(&fs->tag_set, &swim3_mq_ops, 2, + BLK_MQ_F_SHOULD_MERGE); + if (IS_ERR(disk->queue)) { + rc = PTR_ERR(disk->queue); + disk->queue = NULL; + goto out_put_disk; } + blk_queue_bounce_limit(disk->queue, BLK_BOUNCE_HIGH); + disk->queue->queuedata = fs; + + rc = swim3_add_device(mdev, floppy_count); + if (rc) + goto out_cleanup_queue; disk->major = FLOPPY_MAJOR; - disk->first_minor = index; + disk->first_minor = floppy_count; disk->fops = &floppy_fops; - disk->private_data = &floppy_states[index]; + disk->private_data = fs; disk->flags |= GENHD_FL_REMOVABLE; - sprintf(disk->disk_name, "fd%d", index); + sprintf(disk->disk_name, "fd%d", floppy_count); set_capacity(disk, 2880); add_disk(disk); + disks[floppy_count++] = disk; return 0; + +out_cleanup_queue: + blk_cleanup_queue(disk->queue); + disk->queue = NULL; + blk_mq_free_tag_set(&fs->tag_set); +out_put_disk: + put_disk(disk); +out_unregister: + if (floppy_count == 0) + unregister_blkdev(FLOPPY_MAJOR, "fd"); + return rc; } static const struct of_device_id swim3_match[] = diff --git a/drivers/block/sx8.c b/drivers/block/sx8.c index 4d90e5eba2f5..064b8c5c7a32 100644 --- a/drivers/block/sx8.c +++ b/drivers/block/sx8.c @@ -16,7 +16,7 @@ #include <linux/pci.h> #include <linux/slab.h> #include <linux/spinlock.h> -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/sched.h> #include <linux/interrupt.h> #include <linux/compiler.h> @@ -197,7 +197,6 @@ enum { FL_NON_RAID = FW_VER_NON_RAID, FL_4PORT = FW_VER_4PORT, FL_FW_VER_MASK = (FW_VER_NON_RAID | FW_VER_4PORT), - FL_DAC = (1 << 16), FL_DYN_MAJOR = (1 << 17), }; @@ -244,6 +243,7 @@ struct carm_port { unsigned int port_no; struct gendisk *disk; struct carm_host *host; + struct blk_mq_tag_set tag_set; /* attached device characteristics */ u64 capacity; @@ -279,6 +279,7 @@ struct carm_host { unsigned int state; u32 fw_ver; + struct blk_mq_tag_set tag_set; struct request_queue *oob_q; unsigned int n_oob; @@ -750,7 +751,7 @@ static inline void carm_end_request_queued(struct carm_host *host, struct request *req = crq->rq; int rc; - __blk_end_request_all(req, error); + blk_mq_end_request(req, error); rc = carm_put_request(host, crq); assert(rc == 0); @@ -760,7 +761,7 @@ static inline void carm_push_q (struct carm_host *host, struct request_queue *q) { unsigned int idx = host->wait_q_prod % CARM_MAX_WAIT_Q; - blk_stop_queue(q); + blk_mq_stop_hw_queues(q); VPRINTK("STOPPED QUEUE %p\n", q); host->wait_q[idx] = q; @@ -785,7 +786,7 @@ static inline void carm_round_robin(struct carm_host *host) { struct request_queue *q = carm_pop_q(host); if (q) { - blk_start_queue(q); + blk_mq_start_hw_queues(q); VPRINTK("STARTED QUEUE %p\n", q); } } @@ -802,82 +803,86 @@ static inline void carm_end_rq(struct carm_host *host, struct carm_request *crq, } } -static void carm_oob_rq_fn(struct request_queue *q) +static blk_status_t carm_oob_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { + struct request_queue *q = hctx->queue; struct carm_host *host = q->queuedata; struct carm_request *crq; - struct request *rq; int rc; - while (1) { - DPRINTK("get req\n"); - rq = blk_fetch_request(q); - if (!rq) - break; + blk_mq_start_request(bd->rq); - crq = rq->special; - assert(crq != NULL); - assert(crq->rq == rq); + spin_lock_irq(&host->lock); - crq->n_elem = 0; + crq = bd->rq->special; + assert(crq != NULL); + assert(crq->rq == bd->rq); - DPRINTK("send req\n"); - rc = carm_send_msg(host, crq); - if (rc) { - blk_requeue_request(q, rq); - carm_push_q(host, q); - return; /* call us again later, eventually */ - } + crq->n_elem = 0; + + DPRINTK("send req\n"); + rc = carm_send_msg(host, crq); + if (rc) { + carm_push_q(host, q); + spin_unlock_irq(&host->lock); + return BLK_STS_DEV_RESOURCE; } + + spin_unlock_irq(&host->lock); + return BLK_STS_OK; } -static void carm_rq_fn(struct request_queue *q) +static blk_status_t carm_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { + struct request_queue *q = hctx->queue; struct carm_port *port = q->queuedata; struct carm_host *host = port->host; struct carm_msg_rw *msg; struct carm_request *crq; - struct request *rq; + struct request *rq = bd->rq; struct scatterlist *sg; int writing = 0, pci_dir, i, n_elem, rc; u32 tmp; unsigned int msg_size; -queue_one_request: - VPRINTK("get req\n"); - rq = blk_peek_request(q); - if (!rq) - return; + blk_mq_start_request(rq); + + spin_lock_irq(&host->lock); crq = carm_get_request(host); if (!crq) { carm_push_q(host, q); - return; /* call us again later, eventually */ + spin_unlock_irq(&host->lock); + return BLK_STS_DEV_RESOURCE; } crq->rq = rq; - blk_start_request(rq); - if (rq_data_dir(rq) == WRITE) { writing = 1; - pci_dir = PCI_DMA_TODEVICE; + pci_dir = DMA_TO_DEVICE; } else { - pci_dir = PCI_DMA_FROMDEVICE; + pci_dir = DMA_FROM_DEVICE; } /* get scatterlist from block layer */ sg = &crq->sg[0]; n_elem = blk_rq_map_sg(q, rq, sg); if (n_elem <= 0) { + /* request with no s/g entries? */ carm_end_rq(host, crq, BLK_STS_IOERR); - return; /* request with no s/g entries? */ + spin_unlock_irq(&host->lock); + return BLK_STS_IOERR; } /* map scatterlist to PCI bus addresses */ - n_elem = pci_map_sg(host->pdev, sg, n_elem, pci_dir); + n_elem = dma_map_sg(&host->pdev->dev, sg, n_elem, pci_dir); if (n_elem <= 0) { + /* request with no s/g entries? */ carm_end_rq(host, crq, BLK_STS_IOERR); - return; /* request with no s/g entries? */ + spin_unlock_irq(&host->lock); + return BLK_STS_IOERR; } crq->n_elem = n_elem; crq->port = port; @@ -927,12 +932,13 @@ queue_one_request: rc = carm_send_msg(host, crq); if (rc) { carm_put_request(host, crq); - blk_requeue_request(q, rq); carm_push_q(host, q); - return; /* call us again later, eventually */ + spin_unlock_irq(&host->lock); + return BLK_STS_DEV_RESOURCE; } - goto queue_one_request; + spin_unlock_irq(&host->lock); + return BLK_STS_OK; } static void carm_handle_array_info(struct carm_host *host, @@ -1052,11 +1058,11 @@ static inline void carm_handle_rw(struct carm_host *host, VPRINTK("ENTER\n"); if (rq_data_dir(crq->rq) == WRITE) - pci_dir = PCI_DMA_TODEVICE; + pci_dir = DMA_TO_DEVICE; else - pci_dir = PCI_DMA_FROMDEVICE; + pci_dir = DMA_FROM_DEVICE; - pci_unmap_sg(host->pdev, &crq->sg[0], crq->n_elem, pci_dir); + dma_unmap_sg(&host->pdev->dev, &crq->sg[0], crq->n_elem, pci_dir); carm_end_rq(host, crq, error); } @@ -1485,6 +1491,14 @@ static int carm_init_host(struct carm_host *host) return 0; } +static const struct blk_mq_ops carm_oob_mq_ops = { + .queue_rq = carm_oob_queue_rq, +}; + +static const struct blk_mq_ops carm_mq_ops = { + .queue_rq = carm_queue_rq, +}; + static int carm_init_disks(struct carm_host *host) { unsigned int i; @@ -1513,9 +1527,10 @@ static int carm_init_disks(struct carm_host *host) disk->fops = &carm_bd_ops; disk->private_data = port; - q = blk_init_queue(carm_rq_fn, &host->lock); - if (!q) { - rc = -ENOMEM; + q = blk_mq_init_sq_queue(&port->tag_set, &carm_mq_ops, + max_queue, BLK_MQ_F_SHOULD_MERGE); + if (IS_ERR(q)) { + rc = PTR_ERR(q); break; } disk->queue = q; @@ -1533,14 +1548,18 @@ static void carm_free_disks(struct carm_host *host) unsigned int i; for (i = 0; i < CARM_MAX_PORTS; i++) { - struct gendisk *disk = host->port[i].disk; + struct carm_port *port = &host->port[i]; + struct gendisk *disk = port->disk; + if (disk) { struct request_queue *q = disk->queue; if (disk->flags & GENHD_FL_UP) del_gendisk(disk); - if (q) + if (q) { + blk_mq_free_tag_set(&port->tag_set); blk_cleanup_queue(q); + } put_disk(disk); } } @@ -1548,8 +1567,8 @@ static void carm_free_disks(struct carm_host *host) static int carm_init_shm(struct carm_host *host) { - host->shm = pci_alloc_consistent(host->pdev, CARM_SHM_SIZE, - &host->shm_dma); + host->shm = dma_alloc_coherent(&host->pdev->dev, CARM_SHM_SIZE, + &host->shm_dma, GFP_KERNEL); if (!host->shm) return -ENOMEM; @@ -1565,7 +1584,6 @@ static int carm_init_shm(struct carm_host *host) static int carm_init_one (struct pci_dev *pdev, const struct pci_device_id *ent) { struct carm_host *host; - unsigned int pci_dac; int rc; struct request_queue *q; unsigned int i; @@ -1580,28 +1598,12 @@ static int carm_init_one (struct pci_dev *pdev, const struct pci_device_id *ent) if (rc) goto err_out; -#ifdef IF_64BIT_DMA_IS_POSSIBLE /* grrrr... */ - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); - if (!rc) { - rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); - if (rc) { - printk(KERN_ERR DRV_NAME "(%s): consistent DMA mask failure\n", - pci_name(pdev)); - goto err_out_regions; - } - pci_dac = 1; - } else { -#endif - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); - if (rc) { - printk(KERN_ERR DRV_NAME "(%s): DMA mask failure\n", - pci_name(pdev)); - goto err_out_regions; - } - pci_dac = 0; -#ifdef IF_64BIT_DMA_IS_POSSIBLE /* grrrr... */ + rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); + if (rc) { + printk(KERN_ERR DRV_NAME "(%s): DMA mask failure\n", + pci_name(pdev)); + goto err_out_regions; } -#endif host = kzalloc(sizeof(*host), GFP_KERNEL); if (!host) { @@ -1612,7 +1614,6 @@ static int carm_init_one (struct pci_dev *pdev, const struct pci_device_id *ent) } host->pdev = pdev; - host->flags = pci_dac ? FL_DAC : 0; spin_lock_init(&host->lock); INIT_WORK(&host->fsm_task, carm_fsm_task); init_completion(&host->probe_comp); @@ -1636,12 +1637,13 @@ static int carm_init_one (struct pci_dev *pdev, const struct pci_device_id *ent) goto err_out_iounmap; } - q = blk_init_queue(carm_oob_rq_fn, &host->lock); - if (!q) { + q = blk_mq_init_sq_queue(&host->tag_set, &carm_oob_mq_ops, 1, + BLK_MQ_F_NO_SCHED); + if (IS_ERR(q)) { printk(KERN_ERR DRV_NAME "(%s): OOB queue alloc failure\n", pci_name(pdev)); - rc = -ENOMEM; - goto err_out_pci_free; + rc = PTR_ERR(q); + goto err_out_dma_free; } host->oob_q = q; q->queuedata = host; @@ -1705,8 +1707,9 @@ err_out_free_majors: else if (host->major == 161) clear_bit(1, &carm_major_alloc); blk_cleanup_queue(host->oob_q); -err_out_pci_free: - pci_free_consistent(pdev, CARM_SHM_SIZE, host->shm, host->shm_dma); + blk_mq_free_tag_set(&host->tag_set); +err_out_dma_free: + dma_free_coherent(&pdev->dev, CARM_SHM_SIZE, host->shm, host->shm_dma); err_out_iounmap: iounmap(host->mmio); err_out_kfree: @@ -1736,7 +1739,8 @@ static void carm_remove_one (struct pci_dev *pdev) else if (host->major == 161) clear_bit(1, &carm_major_alloc); blk_cleanup_queue(host->oob_q); - pci_free_consistent(pdev, CARM_SHM_SIZE, host->shm, host->shm_dma); + blk_mq_free_tag_set(&host->tag_set); + dma_free_coherent(&pdev->dev, CARM_SHM_SIZE, host->shm, host->shm_dma); iounmap(host->mmio); kfree(host); pci_release_regions(pdev); diff --git a/drivers/block/umem.c b/drivers/block/umem.c index 5c7fb8cc4149..be3e3ab79950 100644 --- a/drivers/block/umem.c +++ b/drivers/block/umem.c @@ -363,12 +363,12 @@ static int add_bio(struct cardinfo *card) vec = bio_iter_iovec(bio, card->current_iter); - dma_handle = pci_map_page(card->dev, + dma_handle = dma_map_page(&card->dev->dev, vec.bv_page, vec.bv_offset, vec.bv_len, bio_op(bio) == REQ_OP_READ ? - PCI_DMA_FROMDEVICE : PCI_DMA_TODEVICE); + DMA_FROM_DEVICE : DMA_TO_DEVICE); p = &card->mm_pages[card->Ready]; desc = &p->desc[p->cnt]; @@ -421,7 +421,7 @@ static void process_page(unsigned long data) struct cardinfo *card = (struct cardinfo *)data; unsigned int dma_status = card->dma_status; - spin_lock_bh(&card->lock); + spin_lock(&card->lock); if (card->Active < 0) goto out_unlock; page = &card->mm_pages[card->Active]; @@ -448,10 +448,10 @@ static void process_page(unsigned long data) page->iter = page->bio->bi_iter; } - pci_unmap_page(card->dev, desc->data_dma_handle, + dma_unmap_page(&card->dev->dev, desc->data_dma_handle, vec.bv_len, (control & DMASCR_TRANSFER_READ) ? - PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE); + DMA_TO_DEVICE : DMA_FROM_DEVICE); if (control & DMASCR_HARD_ERROR) { /* error */ bio->bi_status = BLK_STS_IOERR; @@ -496,7 +496,7 @@ static void process_page(unsigned long data) mm_start_io(card); } out_unlock: - spin_unlock_bh(&card->lock); + spin_unlock(&card->lock); while (return_bio) { struct bio *bio = return_bio; @@ -817,8 +817,8 @@ static int mm_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) dev_printk(KERN_INFO, &dev->dev, "Micro Memory(tm) controller found (PCI Mem Module (Battery Backup))\n"); - if (pci_set_dma_mask(dev, DMA_BIT_MASK(64)) && - pci_set_dma_mask(dev, DMA_BIT_MASK(32))) { + if (dma_set_mask(&dev->dev, DMA_BIT_MASK(64)) && + dma_set_mask(&dev->dev, DMA_BIT_MASK(32))) { dev_printk(KERN_WARNING, &dev->dev, "NO suitable DMA found\n"); return -ENOMEM; } @@ -871,12 +871,10 @@ static int mm_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) goto failed_magic; } - card->mm_pages[0].desc = pci_alloc_consistent(card->dev, - PAGE_SIZE * 2, - &card->mm_pages[0].page_dma); - card->mm_pages[1].desc = pci_alloc_consistent(card->dev, - PAGE_SIZE * 2, - &card->mm_pages[1].page_dma); + card->mm_pages[0].desc = dma_alloc_coherent(&card->dev->dev, + PAGE_SIZE * 2, &card->mm_pages[0].page_dma, GFP_KERNEL); + card->mm_pages[1].desc = dma_alloc_coherent(&card->dev->dev, + PAGE_SIZE * 2, &card->mm_pages[1].page_dma, GFP_KERNEL); if (card->mm_pages[0].desc == NULL || card->mm_pages[1].desc == NULL) { dev_printk(KERN_ERR, &card->dev->dev, "alloc failed\n"); @@ -1002,13 +1000,13 @@ static int mm_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) failed_req_irq: failed_alloc: if (card->mm_pages[0].desc) - pci_free_consistent(card->dev, PAGE_SIZE*2, - card->mm_pages[0].desc, - card->mm_pages[0].page_dma); + dma_free_coherent(&card->dev->dev, PAGE_SIZE * 2, + card->mm_pages[0].desc, + card->mm_pages[0].page_dma); if (card->mm_pages[1].desc) - pci_free_consistent(card->dev, PAGE_SIZE*2, - card->mm_pages[1].desc, - card->mm_pages[1].page_dma); + dma_free_coherent(&card->dev->dev, PAGE_SIZE * 2, + card->mm_pages[1].desc, + card->mm_pages[1].page_dma); failed_magic: iounmap(card->csr_remap); failed_remap_csr: @@ -1027,11 +1025,11 @@ static void mm_pci_remove(struct pci_dev *dev) iounmap(card->csr_remap); if (card->mm_pages[0].desc) - pci_free_consistent(card->dev, PAGE_SIZE*2, + dma_free_coherent(&card->dev->dev, PAGE_SIZE * 2, card->mm_pages[0].desc, card->mm_pages[0].page_dma); if (card->mm_pages[1].desc) - pci_free_consistent(card->dev, PAGE_SIZE*2, + dma_free_coherent(&card->dev->dev, PAGE_SIZE * 2, card->mm_pages[1].desc, card->mm_pages[1].page_dma); blk_cleanup_queue(card->queue); diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 23752dc99b00..086c6bb12baa 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -351,8 +351,8 @@ static int minor_to_index(int minor) return minor >> PART_BITS; } -static ssize_t virtblk_serial_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t serial_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct gendisk *disk = dev_to_disk(dev); int err; @@ -371,7 +371,7 @@ static ssize_t virtblk_serial_show(struct device *dev, return err; } -static DEVICE_ATTR(serial, 0444, virtblk_serial_show, NULL); +static DEVICE_ATTR_RO(serial); /* The queue's logical block size must be set before calling this */ static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize) @@ -545,8 +545,8 @@ static const char *const virtblk_cache_types[] = { }; static ssize_t -virtblk_cache_type_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) +cache_type_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) { struct gendisk *disk = dev_to_disk(dev); struct virtio_blk *vblk = disk->private_data; @@ -564,8 +564,7 @@ virtblk_cache_type_store(struct device *dev, struct device_attribute *attr, } static ssize_t -virtblk_cache_type_show(struct device *dev, struct device_attribute *attr, - char *buf) +cache_type_show(struct device *dev, struct device_attribute *attr, char *buf) { struct gendisk *disk = dev_to_disk(dev); struct virtio_blk *vblk = disk->private_data; @@ -575,12 +574,38 @@ virtblk_cache_type_show(struct device *dev, struct device_attribute *attr, return snprintf(buf, 40, "%s\n", virtblk_cache_types[writeback]); } -static const struct device_attribute dev_attr_cache_type_ro = - __ATTR(cache_type, 0444, - virtblk_cache_type_show, NULL); -static const struct device_attribute dev_attr_cache_type_rw = - __ATTR(cache_type, 0644, - virtblk_cache_type_show, virtblk_cache_type_store); +static DEVICE_ATTR_RW(cache_type); + +static struct attribute *virtblk_attrs[] = { + &dev_attr_serial.attr, + &dev_attr_cache_type.attr, + NULL, +}; + +static umode_t virtblk_attrs_are_visible(struct kobject *kobj, + struct attribute *a, int n) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct gendisk *disk = dev_to_disk(dev); + struct virtio_blk *vblk = disk->private_data; + struct virtio_device *vdev = vblk->vdev; + + if (a == &dev_attr_cache_type.attr && + !virtio_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE)) + return S_IRUGO; + + return a->mode; +} + +static const struct attribute_group virtblk_attr_group = { + .attrs = virtblk_attrs, + .is_visible = virtblk_attrs_are_visible, +}; + +static const struct attribute_group *virtblk_attr_groups[] = { + &virtblk_attr_group, + NULL, +}; static int virtblk_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, unsigned int numa_node) @@ -780,24 +805,9 @@ static int virtblk_probe(struct virtio_device *vdev) virtblk_update_capacity(vblk, false); virtio_device_ready(vdev); - device_add_disk(&vdev->dev, vblk->disk); - err = device_create_file(disk_to_dev(vblk->disk), &dev_attr_serial); - if (err) - goto out_del_disk; - - if (virtio_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE)) - err = device_create_file(disk_to_dev(vblk->disk), - &dev_attr_cache_type_rw); - else - err = device_create_file(disk_to_dev(vblk->disk), - &dev_attr_cache_type_ro); - if (err) - goto out_del_disk; + device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups); return 0; -out_del_disk: - del_gendisk(vblk->disk); - blk_cleanup_queue(vblk->disk->queue); out_free_tags: blk_mq_free_tag_set(&vblk->tag_set); out_put_disk: diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c index 429d20131c7e..9eea83ae01c6 100644 --- a/drivers/block/xen-blkfront.c +++ b/drivers/block/xen-blkfront.c @@ -2420,7 +2420,7 @@ static void blkfront_connect(struct blkfront_info *info) for (i = 0; i < info->nr_rings; i++) kick_pending_request_queues(&info->rinfo[i]); - device_add_disk(&info->xbdev->dev, info->gd); + device_add_disk(&info->xbdev->dev, info->gd, NULL); info->is_ready = 1; return; diff --git a/drivers/block/xsysace.c b/drivers/block/xsysace.c index c24589414c75..87ccef4bd69e 100644 --- a/drivers/block/xsysace.c +++ b/drivers/block/xsysace.c @@ -88,7 +88,7 @@ #include <linux/kernel.h> #include <linux/delay.h> #include <linux/slab.h> -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/mutex.h> #include <linux/ata.h> #include <linux/hdreg.h> @@ -209,6 +209,8 @@ struct ace_device { struct device *dev; struct request_queue *queue; struct gendisk *gd; + struct blk_mq_tag_set tag_set; + struct list_head rq_list; /* Inserted CF card parameters */ u16 cf_id[ATA_ID_WORDS]; @@ -462,18 +464,26 @@ static inline void ace_fsm_yieldirq(struct ace_device *ace) ace->fsm_continue_flag = 0; } +static bool ace_has_next_request(struct request_queue *q) +{ + struct ace_device *ace = q->queuedata; + + return !list_empty(&ace->rq_list); +} + /* Get the next read/write request; ending requests that we don't handle */ static struct request *ace_get_next_request(struct request_queue *q) { - struct request *req; + struct ace_device *ace = q->queuedata; + struct request *rq; - while ((req = blk_peek_request(q)) != NULL) { - if (!blk_rq_is_passthrough(req)) - break; - blk_start_request(req); - __blk_end_request_all(req, BLK_STS_IOERR); + rq = list_first_entry_or_null(&ace->rq_list, struct request, queuelist); + if (rq) { + list_del_init(&rq->queuelist); + blk_mq_start_request(rq); } - return req; + + return NULL; } static void ace_fsm_dostate(struct ace_device *ace) @@ -499,11 +509,11 @@ static void ace_fsm_dostate(struct ace_device *ace) /* Drop all in-flight and pending requests */ if (ace->req) { - __blk_end_request_all(ace->req, BLK_STS_IOERR); + blk_mq_end_request(ace->req, BLK_STS_IOERR); ace->req = NULL; } - while ((req = blk_fetch_request(ace->queue)) != NULL) - __blk_end_request_all(req, BLK_STS_IOERR); + while ((req = ace_get_next_request(ace->queue)) != NULL) + blk_mq_end_request(req, BLK_STS_IOERR); /* Drop back to IDLE state and notify waiters */ ace->fsm_state = ACE_FSM_STATE_IDLE; @@ -517,7 +527,7 @@ static void ace_fsm_dostate(struct ace_device *ace) switch (ace->fsm_state) { case ACE_FSM_STATE_IDLE: /* See if there is anything to do */ - if (ace->id_req_count || ace_get_next_request(ace->queue)) { + if (ace->id_req_count || ace_has_next_request(ace->queue)) { ace->fsm_iter_num++; ace->fsm_state = ACE_FSM_STATE_REQ_LOCK; mod_timer(&ace->stall_timer, jiffies + HZ); @@ -651,7 +661,6 @@ static void ace_fsm_dostate(struct ace_device *ace) ace->fsm_state = ACE_FSM_STATE_IDLE; break; } - blk_start_request(req); /* Okay, it's a data request, set it up for transfer */ dev_dbg(ace->dev, @@ -728,7 +737,8 @@ static void ace_fsm_dostate(struct ace_device *ace) } /* bio finished; is there another one? */ - if (__blk_end_request_cur(ace->req, BLK_STS_OK)) { + if (blk_update_request(ace->req, BLK_STS_OK, + blk_rq_cur_bytes(ace->req))) { /* dev_dbg(ace->dev, "next block; h=%u c=%u\n", * blk_rq_sectors(ace->req), * blk_rq_cur_sectors(ace->req)); @@ -854,17 +864,23 @@ static irqreturn_t ace_interrupt(int irq, void *dev_id) /* --------------------------------------------------------------------- * Block ops */ -static void ace_request(struct request_queue * q) +static blk_status_t ace_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { - struct request *req; - struct ace_device *ace; - - req = ace_get_next_request(q); + struct ace_device *ace = hctx->queue->queuedata; + struct request *req = bd->rq; - if (req) { - ace = req->rq_disk->private_data; - tasklet_schedule(&ace->fsm_tasklet); + if (blk_rq_is_passthrough(req)) { + blk_mq_start_request(req); + return BLK_STS_IOERR; } + + spin_lock_irq(&ace->lock); + list_add_tail(&req->queuelist, &ace->rq_list); + spin_unlock_irq(&ace->lock); + + tasklet_schedule(&ace->fsm_tasklet); + return BLK_STS_OK; } static unsigned int ace_check_events(struct gendisk *gd, unsigned int clearing) @@ -957,6 +973,10 @@ static const struct block_device_operations ace_fops = { .getgeo = ace_getgeo, }; +static const struct blk_mq_ops ace_mq_ops = { + .queue_rq = ace_queue_rq, +}; + /* -------------------------------------------------------------------- * SystemACE device setup/teardown code */ @@ -972,6 +992,7 @@ static int ace_setup(struct ace_device *ace) spin_lock_init(&ace->lock); init_completion(&ace->id_completion); + INIT_LIST_HEAD(&ace->rq_list); /* * Map the device @@ -989,9 +1010,15 @@ static int ace_setup(struct ace_device *ace) /* * Initialize the request queue */ - ace->queue = blk_init_queue(ace_request, &ace->lock); - if (ace->queue == NULL) + ace->queue = blk_mq_init_sq_queue(&ace->tag_set, &ace_mq_ops, 2, + BLK_MQ_F_SHOULD_MERGE); + if (IS_ERR(ace->queue)) { + rc = PTR_ERR(ace->queue); + ace->queue = NULL; goto err_blk_initq; + } + ace->queue->queuedata = ace; + blk_queue_logical_block_size(ace->queue, 512); blk_queue_bounce_limit(ace->queue, BLK_BOUNCE_HIGH); @@ -1066,6 +1093,7 @@ err_read: put_disk(ace->gd); err_alloc_disk: blk_cleanup_queue(ace->queue); + blk_mq_free_tag_set(&ace->tag_set); err_blk_initq: iounmap(ace->baseaddr); err_ioremap: @@ -1081,8 +1109,10 @@ static void ace_teardown(struct ace_device *ace) put_disk(ace->gd); } - if (ace->queue) + if (ace->queue) { blk_cleanup_queue(ace->queue); + blk_mq_free_tag_set(&ace->tag_set); + } tasklet_kill(&ace->fsm_tasklet); diff --git a/drivers/block/z2ram.c b/drivers/block/z2ram.c index d0c5bc4e0703..1106c076fa4b 100644 --- a/drivers/block/z2ram.c +++ b/drivers/block/z2ram.c @@ -31,7 +31,7 @@ #include <linux/vmalloc.h> #include <linux/init.h> #include <linux/module.h> -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/bitops.h> #include <linux/mutex.h> #include <linux/slab.h> @@ -66,43 +66,44 @@ static DEFINE_SPINLOCK(z2ram_lock); static struct gendisk *z2ram_gendisk; -static void do_z2_request(struct request_queue *q) +static blk_status_t z2_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { - struct request *req; - - req = blk_fetch_request(q); - while (req) { - unsigned long start = blk_rq_pos(req) << 9; - unsigned long len = blk_rq_cur_bytes(req); - blk_status_t err = BLK_STS_OK; - - if (start + len > z2ram_size) { - pr_err(DEVICE_NAME ": bad access: block=%llu, " - "count=%u\n", - (unsigned long long)blk_rq_pos(req), - blk_rq_cur_sectors(req)); - err = BLK_STS_IOERR; - goto done; - } - while (len) { - unsigned long addr = start & Z2RAM_CHUNKMASK; - unsigned long size = Z2RAM_CHUNKSIZE - addr; - void *buffer = bio_data(req->bio); - - if (len < size) - size = len; - addr += z2ram_map[ start >> Z2RAM_CHUNKSHIFT ]; - if (rq_data_dir(req) == READ) - memcpy(buffer, (char *)addr, size); - else - memcpy((char *)addr, buffer, size); - start += size; - len -= size; - } - done: - if (!__blk_end_request_cur(req, err)) - req = blk_fetch_request(q); + struct request *req = bd->rq; + unsigned long start = blk_rq_pos(req) << 9; + unsigned long len = blk_rq_cur_bytes(req); + + blk_mq_start_request(req); + + if (start + len > z2ram_size) { + pr_err(DEVICE_NAME ": bad access: block=%llu, " + "count=%u\n", + (unsigned long long)blk_rq_pos(req), + blk_rq_cur_sectors(req)); + return BLK_STS_IOERR; + } + + spin_lock_irq(&z2ram_lock); + + while (len) { + unsigned long addr = start & Z2RAM_CHUNKMASK; + unsigned long size = Z2RAM_CHUNKSIZE - addr; + void *buffer = bio_data(req->bio); + + if (len < size) + size = len; + addr += z2ram_map[ start >> Z2RAM_CHUNKSHIFT ]; + if (rq_data_dir(req) == READ) + memcpy(buffer, (char *)addr, size); + else + memcpy((char *)addr, buffer, size); + start += size; + len -= size; } + + spin_unlock_irq(&z2ram_lock); + blk_mq_end_request(req, BLK_STS_OK); + return BLK_STS_OK; } static void @@ -337,6 +338,11 @@ static struct kobject *z2_find(dev_t dev, int *part, void *data) } static struct request_queue *z2_queue; +static struct blk_mq_tag_set tag_set; + +static const struct blk_mq_ops z2_mq_ops = { + .queue_rq = z2_queue_rq, +}; static int __init z2_init(void) @@ -355,9 +361,13 @@ z2_init(void) if (!z2ram_gendisk) goto out_disk; - z2_queue = blk_init_queue(do_z2_request, &z2ram_lock); - if (!z2_queue) + z2_queue = blk_mq_init_sq_queue(&tag_set, &z2_mq_ops, 16, + BLK_MQ_F_SHOULD_MERGE); + if (IS_ERR(z2_queue)) { + ret = PTR_ERR(z2_queue); + z2_queue = NULL; goto out_queue; + } z2ram_gendisk->major = Z2RAM_MAJOR; z2ram_gendisk->first_minor = 0; @@ -387,6 +397,7 @@ static void __exit z2_exit(void) del_gendisk(z2ram_gendisk); put_disk(z2ram_gendisk); blk_cleanup_queue(z2_queue); + blk_mq_free_tag_set(&tag_set); if ( current_device != -1 ) { diff --git a/drivers/block/zram/Kconfig b/drivers/block/zram/Kconfig index 635235759a0a..fcd055457364 100644 --- a/drivers/block/zram/Kconfig +++ b/drivers/block/zram/Kconfig @@ -3,7 +3,6 @@ config ZRAM tristate "Compressed RAM block device support" depends on BLOCK && SYSFS && ZSMALLOC && CRYPTO select CRYPTO_LZO - default n help Creates virtual block devices called /dev/zramX (X = 0, 1, ...). Pages written to these disks are compressed and stored in memory @@ -18,7 +17,6 @@ config ZRAM config ZRAM_WRITEBACK bool "Write back incompressible page to backing device" depends on ZRAM - default n help With incompressible page, there is no memory saving to keep it in memory. Instead, write it out to backing device. diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index a1d6b5597c17..4879595200e1 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1636,6 +1636,11 @@ static const struct attribute_group zram_disk_attr_group = { .attrs = zram_disk_attrs, }; +static const struct attribute_group *zram_disk_attr_groups[] = { + &zram_disk_attr_group, + NULL, +}; + /* * Allocate and initialize new zram device. the function returns * '>= 0' device_id upon success, and negative value otherwise. @@ -1716,24 +1721,14 @@ static int zram_add(void) zram->disk->queue->backing_dev_info->capabilities |= (BDI_CAP_STABLE_WRITES | BDI_CAP_SYNCHRONOUS_IO); - add_disk(zram->disk); - - ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj, - &zram_disk_attr_group); - if (ret < 0) { - pr_err("Error creating sysfs group for device %d\n", - device_id); - goto out_free_disk; - } + device_add_disk(NULL, zram->disk, zram_disk_attr_groups); + strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor)); zram_debugfs_register(zram); pr_info("Added device: %s\n", zram->disk->disk_name); return device_id; -out_free_disk: - del_gendisk(zram->disk); - put_disk(zram->disk); out_free_queue: blk_cleanup_queue(queue); out_free_idr: @@ -1762,15 +1757,6 @@ static int zram_remove(struct zram *zram) mutex_unlock(&bdev->bd_mutex); zram_debugfs_unregister(zram); - /* - * Remove sysfs first, so no one will perform a disksize - * store while we destroy the devices. This also helps during - * hot_remove -- zram_reset_device() is the last holder of - * ->init_lock, no later/concurrent disksize_store() or any - * other sysfs handlers are possible. - */ - sysfs_remove_group(&disk_to_dev(zram->disk)->kobj, - &zram_disk_attr_group); /* Make sure all the pending I/O are finished */ fsync_bdev(bdev); diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c index a5d5a96479bf..614ecdbb4ab7 100644 --- a/drivers/cdrom/cdrom.c +++ b/drivers/cdrom/cdrom.c @@ -410,10 +410,10 @@ static int cdrom_get_disc_info(struct cdrom_device_info *cdi, * hack to have the capability flags defined const, while we can still * change it here without gcc complaining at every line. */ -#define ENSURE(call, bits) \ -do { \ - if (cdo->call == NULL) \ - *change_capability &= ~(bits); \ +#define ENSURE(cdo, call, bits) \ +do { \ + if (cdo->call == NULL) \ + WARN_ON_ONCE((cdo)->capability & (bits)); \ } while (0) /* @@ -589,7 +589,6 @@ int register_cdrom(struct cdrom_device_info *cdi) { static char banner_printed; const struct cdrom_device_ops *cdo = cdi->ops; - int *change_capability = (int *)&cdo->capability; /* hack */ cd_dbg(CD_OPEN, "entering register_cdrom\n"); @@ -601,16 +600,16 @@ int register_cdrom(struct cdrom_device_info *cdi) cdrom_sysctl_register(); } - ENSURE(drive_status, CDC_DRIVE_STATUS); + ENSURE(cdo, drive_status, CDC_DRIVE_STATUS); if (cdo->check_events == NULL && cdo->media_changed == NULL) - *change_capability = ~(CDC_MEDIA_CHANGED | CDC_SELECT_DISC); - ENSURE(tray_move, CDC_CLOSE_TRAY | CDC_OPEN_TRAY); - ENSURE(lock_door, CDC_LOCK); - ENSURE(select_speed, CDC_SELECT_SPEED); - ENSURE(get_last_session, CDC_MULTI_SESSION); - ENSURE(get_mcn, CDC_MCN); - ENSURE(reset, CDC_RESET); - ENSURE(generic_packet, CDC_GENERIC_PACKET); + WARN_ON_ONCE(cdo->capability & (CDC_MEDIA_CHANGED | CDC_SELECT_DISC)); + ENSURE(cdo, tray_move, CDC_CLOSE_TRAY | CDC_OPEN_TRAY); + ENSURE(cdo, lock_door, CDC_LOCK); + ENSURE(cdo, select_speed, CDC_SELECT_SPEED); + ENSURE(cdo, get_last_session, CDC_MULTI_SESSION); + ENSURE(cdo, get_mcn, CDC_MCN); + ENSURE(cdo, reset, CDC_RESET); + ENSURE(cdo, generic_packet, CDC_GENERIC_PACKET); cdi->mc_flags = 0; cdi->options = CDO_USE_FFLAGS; @@ -2445,7 +2444,7 @@ static int cdrom_ioctl_select_disc(struct cdrom_device_info *cdi, return -ENOSYS; if (arg != CDSL_CURRENT && arg != CDSL_NONE) { - if ((int)arg >= cdi->capacity) + if (arg >= cdi->capacity) return -EINVAL; } diff --git a/drivers/cdrom/gdrom.c b/drivers/cdrom/gdrom.c index ae3a7537cf0f..757e85b81879 100644 --- a/drivers/cdrom/gdrom.c +++ b/drivers/cdrom/gdrom.c @@ -31,12 +31,11 @@ #include <linux/cdrom.h> #include <linux/genhd.h> #include <linux/bio.h> -#include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/interrupt.h> #include <linux/device.h> #include <linux/mutex.h> #include <linux/wait.h> -#include <linux/workqueue.h> #include <linux/platform_device.h> #include <scsi/scsi.h> #include <asm/io.h> @@ -102,11 +101,6 @@ static int gdrom_major; static DECLARE_WAIT_QUEUE_HEAD(command_queue); static DECLARE_WAIT_QUEUE_HEAD(request_queue); -static DEFINE_SPINLOCK(gdrom_lock); -static void gdrom_readdisk_dma(struct work_struct *work); -static DECLARE_WORK(work, gdrom_readdisk_dma); -static LIST_HEAD(gdrom_deferred); - struct gdromtoc { unsigned int entry[99]; unsigned int first, last; @@ -122,6 +116,7 @@ static struct gdrom_unit { char disk_type; struct gdromtoc *toc; struct request_queue *gdrom_rq; + struct blk_mq_tag_set tag_set; } gd; struct gdrom_id { @@ -584,103 +579,83 @@ static int gdrom_set_interrupt_handlers(void) * 9 -> sectors >> 8 * 10 -> sectors */ -static void gdrom_readdisk_dma(struct work_struct *work) +static blk_status_t gdrom_readdisk_dma(struct request *req) { int block, block_cnt; blk_status_t err; struct packet_command *read_command; - struct list_head *elem, *next; - struct request *req; unsigned long timeout; - if (list_empty(&gdrom_deferred)) - return; read_command = kzalloc(sizeof(struct packet_command), GFP_KERNEL); if (!read_command) - return; /* get more memory later? */ + return BLK_STS_RESOURCE; + read_command->cmd[0] = 0x30; read_command->cmd[1] = 0x20; - spin_lock(&gdrom_lock); - list_for_each_safe(elem, next, &gdrom_deferred) { - req = list_entry(elem, struct request, queuelist); - spin_unlock(&gdrom_lock); - block = blk_rq_pos(req)/GD_TO_BLK + GD_SESSION_OFFSET; - block_cnt = blk_rq_sectors(req)/GD_TO_BLK; - __raw_writel(virt_to_phys(bio_data(req->bio)), GDROM_DMA_STARTADDR_REG); - __raw_writel(block_cnt * GDROM_HARD_SECTOR, GDROM_DMA_LENGTH_REG); - __raw_writel(1, GDROM_DMA_DIRECTION_REG); - __raw_writel(1, GDROM_DMA_ENABLE_REG); - read_command->cmd[2] = (block >> 16) & 0xFF; - read_command->cmd[3] = (block >> 8) & 0xFF; - read_command->cmd[4] = block & 0xFF; - read_command->cmd[8] = (block_cnt >> 16) & 0xFF; - read_command->cmd[9] = (block_cnt >> 8) & 0xFF; - read_command->cmd[10] = block_cnt & 0xFF; - /* set for DMA */ - __raw_writeb(1, GDROM_ERROR_REG); - /* other registers */ - __raw_writeb(0, GDROM_SECNUM_REG); - __raw_writeb(0, GDROM_BCL_REG); - __raw_writeb(0, GDROM_BCH_REG); - __raw_writeb(0, GDROM_DSEL_REG); - __raw_writeb(0, GDROM_INTSEC_REG); - /* Wait for registers to reset after any previous activity */ - timeout = jiffies + HZ / 2; - while (gdrom_is_busy() && time_before(jiffies, timeout)) - cpu_relax(); - __raw_writeb(GDROM_COM_PACKET, GDROM_STATUSCOMMAND_REG); - timeout = jiffies + HZ / 2; - /* Wait for packet command to finish */ - while (gdrom_is_busy() && time_before(jiffies, timeout)) - cpu_relax(); - gd.pending = 1; - gd.transfer = 1; - outsw(GDROM_DATA_REG, &read_command->cmd, 6); - timeout = jiffies + HZ / 2; - /* Wait for any pending DMA to finish */ - while (__raw_readb(GDROM_DMA_STATUS_REG) && - time_before(jiffies, timeout)) - cpu_relax(); - /* start transfer */ - __raw_writeb(1, GDROM_DMA_STATUS_REG); - wait_event_interruptible_timeout(request_queue, - gd.transfer == 0, GDROM_DEFAULT_TIMEOUT); - err = gd.transfer ? BLK_STS_IOERR : BLK_STS_OK; - gd.transfer = 0; - gd.pending = 0; - /* now seek to take the request spinlock - * before handling ending the request */ - spin_lock(&gdrom_lock); - list_del_init(&req->queuelist); - __blk_end_request_all(req, err); - } - spin_unlock(&gdrom_lock); + block = blk_rq_pos(req)/GD_TO_BLK + GD_SESSION_OFFSET; + block_cnt = blk_rq_sectors(req)/GD_TO_BLK; + __raw_writel(virt_to_phys(bio_data(req->bio)), GDROM_DMA_STARTADDR_REG); + __raw_writel(block_cnt * GDROM_HARD_SECTOR, GDROM_DMA_LENGTH_REG); + __raw_writel(1, GDROM_DMA_DIRECTION_REG); + __raw_writel(1, GDROM_DMA_ENABLE_REG); + read_command->cmd[2] = (block >> 16) & 0xFF; + read_command->cmd[3] = (block >> 8) & 0xFF; + read_command->cmd[4] = block & 0xFF; + read_command->cmd[8] = (block_cnt >> 16) & 0xFF; + read_command->cmd[9] = (block_cnt >> 8) & 0xFF; + read_command->cmd[10] = block_cnt & 0xFF; + /* set for DMA */ + __raw_writeb(1, GDROM_ERROR_REG); + /* other registers */ + __raw_writeb(0, GDROM_SECNUM_REG); + __raw_writeb(0, GDROM_BCL_REG); + __raw_writeb(0, GDROM_BCH_REG); + __raw_writeb(0, GDROM_DSEL_REG); + __raw_writeb(0, GDROM_INTSEC_REG); + /* Wait for registers to reset after any previous activity */ + timeout = jiffies + HZ / 2; + while (gdrom_is_busy() && time_before(jiffies, timeout)) + cpu_relax(); + __raw_writeb(GDROM_COM_PACKET, GDROM_STATUSCOMMAND_REG); + timeout = jiffies + HZ / 2; + /* Wait for packet command to finish */ + while (gdrom_is_busy() && time_before(jiffies, timeout)) + cpu_relax(); + gd.pending = 1; + gd.transfer = 1; + outsw(GDROM_DATA_REG, &read_command->cmd, 6); + timeout = jiffies + HZ / 2; + /* Wait for any pending DMA to finish */ + while (__raw_readb(GDROM_DMA_STATUS_REG) && + time_before(jiffies, timeout)) + cpu_relax(); + /* start transfer */ + __raw_writeb(1, GDROM_DMA_STATUS_REG); + wait_event_interruptible_timeout(request_queue, + gd.transfer == 0, GDROM_DEFAULT_TIMEOUT); + err = gd.transfer ? BLK_STS_IOERR : BLK_STS_OK; + gd.transfer = 0; + gd.pending = 0; + + blk_mq_end_request(req, err); kfree(read_command); + return BLK_STS_OK; } -static void gdrom_request(struct request_queue *rq) -{ - struct request *req; - - while ((req = blk_fetch_request(rq)) != NULL) { - switch (req_op(req)) { - case REQ_OP_READ: - /* - * Add to list of deferred work and then schedule - * workqueue. - */ - list_add_tail(&req->queuelist, &gdrom_deferred); - schedule_work(&work); - break; - case REQ_OP_WRITE: - pr_notice("Read only device - write request ignored\n"); - __blk_end_request_all(req, BLK_STS_IOERR); - break; - default: - printk(KERN_DEBUG "gdrom: Non-fs request ignored\n"); - __blk_end_request_all(req, BLK_STS_IOERR); - break; - } +static blk_status_t gdrom_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) +{ + blk_mq_start_request(bd->rq); + + switch (req_op(bd->rq)) { + case REQ_OP_READ: + return gdrom_readdisk_dma(bd->rq); + case REQ_OP_WRITE: + pr_notice("Read only device - write request ignored\n"); + return BLK_STS_IOERR; + default: + printk(KERN_DEBUG "gdrom: Non-fs request ignored\n"); + return BLK_STS_IOERR; } } @@ -768,6 +743,10 @@ static int probe_gdrom_setupqueue(void) return gdrom_init_dma_mode(); } +static const struct blk_mq_ops gdrom_mq_ops = { + .queue_rq = gdrom_queue_rq, +}; + /* * register this as a block device and as compliant with the * universal CD Rom driver interface @@ -811,11 +790,15 @@ static int probe_gdrom(struct platform_device *devptr) err = gdrom_set_interrupt_handlers(); if (err) goto probe_fail_cmdirq_register; - gd.gdrom_rq = blk_init_queue(gdrom_request, &gdrom_lock); - if (!gd.gdrom_rq) { - err = -ENOMEM; + + gd.gdrom_rq = blk_mq_init_sq_queue(&gd.tag_set, &gdrom_mq_ops, 1, + BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING); + if (IS_ERR(gd.gdrom_rq)) { + rc = PTR_ERR(gd.gdrom_rq); + gd.gdrom_rq = NULL; goto probe_fail_requestq; } + blk_queue_bounce_limit(gd.gdrom_rq, BLK_BOUNCE_HIGH); err = probe_gdrom_setupqueue(); @@ -832,6 +815,7 @@ static int probe_gdrom(struct platform_device *devptr) probe_fail_toc: blk_cleanup_queue(gd.gdrom_rq); + blk_mq_free_tag_set(&gd.tag_set); probe_fail_requestq: free_irq(HW_EVENT_GDROM_DMA, &gd); free_irq(HW_EVENT_GDROM_CMD, &gd); @@ -849,8 +833,8 @@ probe_fail_no_mem: static int remove_gdrom(struct platform_device *devptr) { - flush_work(&work); blk_cleanup_queue(gd.gdrom_rq); + blk_mq_free_tag_set(&gd.tag_set); free_irq(HW_EVENT_GDROM_CMD, &gd); free_irq(HW_EVENT_GDROM_DMA, &gd); del_gendisk(gd.disk); diff --git a/drivers/clk/sunxi-ng/ccu-sun4i-a10.c b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c index ffa5dac221e4..129ebd2588fd 100644 --- a/drivers/clk/sunxi-ng/ccu-sun4i-a10.c +++ b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c @@ -1434,8 +1434,16 @@ static void __init sun4i_ccu_init(struct device_node *node, return; } - /* Force the PLL-Audio-1x divider to 1 */ val = readl(reg + SUN4I_PLL_AUDIO_REG); + + /* + * Force VCO and PLL bias current to lowest setting. Higher + * settings interfere with sigma-delta modulation and result + * in audible noise and distortions when using SPDIF or I2S. + */ + val &= ~GENMASK(25, 16); + + /* Force the PLL-Audio-1x divider to 1 */ val &= ~GENMASK(29, 26); writel(val | (1 << 26), reg + SUN4I_PLL_AUDIO_REG); diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index d8c7f5750cdb..9a7d4dc00b6e 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -319,6 +319,13 @@ static u64 notrace arm64_858921_read_cntvct_el0(void) } #endif +#ifdef CONFIG_ARM64_ERRATUM_1188873 +static u64 notrace arm64_1188873_read_cntvct_el0(void) +{ + return read_sysreg(cntvct_el0); +} +#endif + #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround); EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround); @@ -408,6 +415,14 @@ static const struct arch_timer_erratum_workaround ool_workarounds[] = { .read_cntvct_el0 = arm64_858921_read_cntvct_el0, }, #endif +#ifdef CONFIG_ARM64_ERRATUM_1188873 + { + .match_type = ate_match_local_cap_id, + .id = (void *)ARM64_WORKAROUND_1188873, + .desc = "ARM erratum 1188873", + .read_cntvct_el0 = arm64_1188873_read_cntvct_el0, + }, +#endif }; typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *, diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c index 5762c3c383f2..ab7c5a937ab0 100644 --- a/drivers/edac/altera_edac.c +++ b/drivers/edac/altera_edac.c @@ -599,7 +599,8 @@ static const struct regmap_config s10_sdram_regmap_cfg = { .volatile_reg = s10_sdram_volatile_reg, .reg_read = s10_protected_reg_read, .reg_write = s10_protected_reg_write, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; static int altr_s10_sdram_probe(struct platform_device *pdev) diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c index 018fcdb353d2..281cf9cbb44c 100644 --- a/drivers/gpu/drm/drm_atomic.c +++ b/drivers/gpu/drm/drm_atomic.c @@ -174,6 +174,11 @@ void drm_atomic_state_default_clear(struct drm_atomic_state *state) state->crtcs[i].state = NULL; state->crtcs[i].old_state = NULL; state->crtcs[i].new_state = NULL; + + if (state->crtcs[i].commit) { + drm_crtc_commit_put(state->crtcs[i].commit); + state->crtcs[i].commit = NULL; + } } for (i = 0; i < config->num_total_plane; i++) { diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c index 80be74df7ba6..1bb4c318bdd4 100644 --- a/drivers/gpu/drm/drm_atomic_helper.c +++ b/drivers/gpu/drm/drm_atomic_helper.c @@ -1408,15 +1408,16 @@ EXPORT_SYMBOL(drm_atomic_helper_wait_for_vblanks); void drm_atomic_helper_wait_for_flip_done(struct drm_device *dev, struct drm_atomic_state *old_state) { - struct drm_crtc_state *new_crtc_state; struct drm_crtc *crtc; int i; - for_each_new_crtc_in_state(old_state, crtc, new_crtc_state, i) { - struct drm_crtc_commit *commit = new_crtc_state->commit; + for (i = 0; i < dev->mode_config.num_crtc; i++) { + struct drm_crtc_commit *commit = old_state->crtcs[i].commit; int ret; - if (!commit) + crtc = old_state->crtcs[i].ptr; + + if (!crtc || !commit) continue; ret = wait_for_completion_timeout(&commit->flip_done, 10 * HZ); @@ -1934,6 +1935,9 @@ int drm_atomic_helper_setup_commit(struct drm_atomic_state *state, drm_crtc_commit_get(commit); commit->abort_completion = true; + + state->crtcs[i].commit = commit; + drm_crtc_commit_get(commit); } for_each_oldnew_connector_in_state(state, conn, old_conn_state, new_conn_state, i) { diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c index bae43938c8f6..9cbe8f5c9aca 100644 --- a/drivers/gpu/drm/drm_crtc.c +++ b/drivers/gpu/drm/drm_crtc.c @@ -567,9 +567,9 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data, struct drm_mode_crtc *crtc_req = data; struct drm_crtc *crtc; struct drm_plane *plane; - struct drm_connector **connector_set = NULL, *connector; - struct drm_framebuffer *fb = NULL; - struct drm_display_mode *mode = NULL; + struct drm_connector **connector_set, *connector; + struct drm_framebuffer *fb; + struct drm_display_mode *mode; struct drm_mode_set set; uint32_t __user *set_connectors_ptr; struct drm_modeset_acquire_ctx ctx; @@ -598,6 +598,10 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data, mutex_lock(&crtc->dev->mode_config.mutex); drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); retry: + connector_set = NULL; + fb = NULL; + mode = NULL; + ret = drm_modeset_lock_all_ctx(crtc->dev, &ctx); if (ret) goto out; diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c index 3c9fc99648b7..ff0bfc65a8c1 100644 --- a/drivers/gpu/drm/drm_edid.c +++ b/drivers/gpu/drm/drm_edid.c @@ -113,6 +113,9 @@ static const struct edid_quirk { /* AEO model 0 reports 8 bpc, but is a 6 bpc panel */ { "AEO", 0, EDID_QUIRK_FORCE_6BPC }, + /* BOE model on HP Pavilion 15-n233sl reports 8 bpc, but is a 6 bpc panel */ + { "BOE", 0x78b, EDID_QUIRK_FORCE_6BPC }, + /* CPT panel of Asus UX303LA reports 8 bpc, but is a 6 bpc panel */ { "CPT", 0x17df, EDID_QUIRK_FORCE_6BPC }, @@ -4279,7 +4282,7 @@ static void drm_parse_ycbcr420_deep_color_info(struct drm_connector *connector, struct drm_hdmi_info *hdmi = &connector->display_info.hdmi; dc_mask = db[7] & DRM_EDID_YCBCR420_DC_MASK; - hdmi->y420_dc_modes |= dc_mask; + hdmi->y420_dc_modes = dc_mask; } static void drm_parse_hdmi_forum_vsdb(struct drm_connector *connector, diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c index 515a7aec57ac..9628dd617826 100644 --- a/drivers/gpu/drm/drm_fb_helper.c +++ b/drivers/gpu/drm/drm_fb_helper.c @@ -1580,6 +1580,25 @@ unlock: } EXPORT_SYMBOL(drm_fb_helper_ioctl); +static bool drm_fb_pixel_format_equal(const struct fb_var_screeninfo *var_1, + const struct fb_var_screeninfo *var_2) +{ + return var_1->bits_per_pixel == var_2->bits_per_pixel && + var_1->grayscale == var_2->grayscale && + var_1->red.offset == var_2->red.offset && + var_1->red.length == var_2->red.length && + var_1->red.msb_right == var_2->red.msb_right && + var_1->green.offset == var_2->green.offset && + var_1->green.length == var_2->green.length && + var_1->green.msb_right == var_2->green.msb_right && + var_1->blue.offset == var_2->blue.offset && + var_1->blue.length == var_2->blue.length && + var_1->blue.msb_right == var_2->blue.msb_right && + var_1->transp.offset == var_2->transp.offset && + var_1->transp.length == var_2->transp.length && + var_1->transp.msb_right == var_2->transp.msb_right; +} + /** * drm_fb_helper_check_var - implementation for &fb_ops.fb_check_var * @var: screeninfo to check @@ -1590,7 +1609,6 @@ int drm_fb_helper_check_var(struct fb_var_screeninfo *var, { struct drm_fb_helper *fb_helper = info->par; struct drm_framebuffer *fb = fb_helper->fb; - int depth; if (var->pixclock != 0 || in_dbg_master()) return -EINVAL; @@ -1610,72 +1628,15 @@ int drm_fb_helper_check_var(struct fb_var_screeninfo *var, return -EINVAL; } - switch (var->bits_per_pixel) { - case 16: - depth = (var->green.length == 6) ? 16 : 15; - break; - case 32: - depth = (var->transp.length > 0) ? 32 : 24; - break; - default: - depth = var->bits_per_pixel; - break; - } - - switch (depth) { - case 8: - var->red.offset = 0; - var->green.offset = 0; - var->blue.offset = 0; - var->red.length = 8; - var->green.length = 8; - var->blue.length = 8; - var->transp.length = 0; - var->transp.offset = 0; - break; - case 15: - var->red.offset = 10; - var->green.offset = 5; - var->blue.offset = 0; - var->red.length = 5; - var->green.length = 5; - var->blue.length = 5; - var->transp.length = 1; - var->transp.offset = 15; - break; - case 16: - var->red.offset = 11; - var->green.offset = 5; - var->blue.offset = 0; - var->red.length = 5; - var->green.length = 6; - var->blue.length = 5; - var->transp.length = 0; - var->transp.offset = 0; - break; - case 24: - var->red.offset = 16; - var->green.offset = 8; - var->blue.offset = 0; - var->red.length = 8; - var->green.length = 8; - var->blue.length = 8; - var->transp.length = 0; - var->transp.offset = 0; - break; - case 32: - var->red.offset = 16; - var->green.offset = 8; - var->blue.offset = 0; - var->red.length = 8; - var->green.length = 8; - var->blue.length = 8; - var->transp.length = 8; - var->transp.offset = 24; - break; - default: + /* + * drm fbdev emulation doesn't support changing the pixel format at all, + * so reject all pixel format changing requests. + */ + if (!drm_fb_pixel_format_equal(var, &info->var)) { + DRM_DEBUG("fbdev emulation doesn't support changing the pixel format\n"); return -EINVAL; } + return 0; } EXPORT_SYMBOL(drm_fb_helper_check_var); diff --git a/drivers/gpu/drm/sun4i/sun4i_dotclock.c b/drivers/gpu/drm/sun4i/sun4i_dotclock.c index e36004fbe453..2a15f2f9271e 100644 --- a/drivers/gpu/drm/sun4i/sun4i_dotclock.c +++ b/drivers/gpu/drm/sun4i/sun4i_dotclock.c @@ -81,9 +81,19 @@ static long sun4i_dclk_round_rate(struct clk_hw *hw, unsigned long rate, int i; for (i = tcon->dclk_min_div; i <= tcon->dclk_max_div; i++) { - unsigned long ideal = rate * i; + u64 ideal = (u64)rate * i; unsigned long rounded; + /* + * ideal has overflowed the max value that can be stored in an + * unsigned long, and every clk operation we might do on a + * truncated u64 value will give us incorrect results. + * Let's just stop there since bigger dividers will result in + * the same overflow issue. + */ + if (ideal > ULONG_MAX) + goto out; + rounded = clk_hw_round_rate(clk_hw_get_parent(hw), ideal); diff --git a/drivers/hwmon/aspeed-pwm-tacho.c b/drivers/hwmon/aspeed-pwm-tacho.c index 5e449eac788a..92de8139d398 100644 --- a/drivers/hwmon/aspeed-pwm-tacho.c +++ b/drivers/hwmon/aspeed-pwm-tacho.c @@ -852,7 +852,7 @@ static int aspeed_create_pwm_cooling(struct device *dev, dev_err(dev, "Property 'cooling-levels' cannot be read.\n"); return ret; } - snprintf(cdev->name, MAX_CDEV_NAME_LEN, "%s%d", child->name, pwm_port); + snprintf(cdev->name, MAX_CDEV_NAME_LEN, "%pOFn%d", child, pwm_port); cdev->tcdev = thermal_of_cooling_device_register(child, cdev->name, diff --git a/drivers/hwmon/asus_atk0110.c b/drivers/hwmon/asus_atk0110.c index a6636fe42189..a7cf00885c5d 100644 --- a/drivers/hwmon/asus_atk0110.c +++ b/drivers/hwmon/asus_atk0110.c @@ -1210,10 +1210,8 @@ static int atk_register_hwmon(struct atk_data *data) data->hwmon_dev = hwmon_device_register_with_groups(dev, "atk0110", data, data->attr_groups); - if (IS_ERR(data->hwmon_dev)) - return PTR_ERR(data->hwmon_dev); - return 0; + return PTR_ERR_OR_ZERO(data->hwmon_dev); } static int atk_probe_if(struct atk_data *data) diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c index 33d51281272b..975c95169884 100644 --- a/drivers/hwmon/hwmon.c +++ b/drivers/hwmon/hwmon.c @@ -24,6 +24,9 @@ #include <linux/string.h> #include <linux/thermal.h> +#define CREATE_TRACE_POINTS +#include <trace/events/hwmon.h> + #define HWMON_ID_PREFIX "hwmon" #define HWMON_ID_FORMAT HWMON_ID_PREFIX "%d" @@ -171,6 +174,13 @@ static int hwmon_thermal_add_sensor(struct device *dev, } #endif /* IS_REACHABLE(CONFIG_THERMAL) && ... */ +static int hwmon_attr_base(enum hwmon_sensor_types type) +{ + if (type == hwmon_in) + return 0; + return 1; +} + /* sysfs attribute management */ static ssize_t hwmon_attr_show(struct device *dev, @@ -185,6 +195,9 @@ static ssize_t hwmon_attr_show(struct device *dev, if (ret < 0) return ret; + trace_hwmon_attr_show(hattr->index + hwmon_attr_base(hattr->type), + hattr->name, val); + return sprintf(buf, "%ld\n", val); } @@ -193,6 +206,7 @@ static ssize_t hwmon_attr_show_string(struct device *dev, char *buf) { struct hwmon_device_attribute *hattr = to_hwmon_attr(devattr); + enum hwmon_sensor_types type = hattr->type; const char *s; int ret; @@ -201,6 +215,9 @@ static ssize_t hwmon_attr_show_string(struct device *dev, if (ret < 0) return ret; + trace_hwmon_attr_show_string(hattr->index + hwmon_attr_base(type), + hattr->name, s); + return sprintf(buf, "%s\n", s); } @@ -221,14 +238,10 @@ static ssize_t hwmon_attr_store(struct device *dev, if (ret < 0) return ret; - return count; -} + trace_hwmon_attr_store(hattr->index + hwmon_attr_base(hattr->type), + hattr->name, val); -static int hwmon_attr_base(enum hwmon_sensor_types type) -{ - if (type == hwmon_in) - return 0; - return 1; + return count; } static bool is_string_attr(enum hwmon_sensor_types type, u32 attr) @@ -356,6 +369,7 @@ static const char * const hwmon_in_attr_templates[] = { [hwmon_in_max_alarm] = "in%d_max_alarm", [hwmon_in_lcrit_alarm] = "in%d_lcrit_alarm", [hwmon_in_crit_alarm] = "in%d_crit_alarm", + [hwmon_in_enable] = "in%d_enable", }; static const char * const hwmon_curr_attr_templates[] = { diff --git a/drivers/hwmon/ibmpowernv.c b/drivers/hwmon/ibmpowernv.c index 83472808c816..0ccca87f5271 100644 --- a/drivers/hwmon/ibmpowernv.c +++ b/drivers/hwmon/ibmpowernv.c @@ -458,9 +458,6 @@ static int populate_attr_groups(struct platform_device *pdev) for_each_child_of_node(opal, np) { const char *label; - if (np->name == NULL) - continue; - type = get_sensor_type(np); if (type == MAX_SENSOR_TYPE) continue; @@ -589,9 +586,6 @@ static int create_device_attrs(struct platform_device *pdev) const char *label; enum sensors type; - if (np->name == NULL) - continue; - type = get_sensor_type(np); if (type == MAX_SENSOR_TYPE) continue; @@ -603,8 +597,8 @@ static int create_device_attrs(struct platform_device *pdev) if (of_property_read_u32(np, "sensor-id", &sensor_id) && of_property_read_u32(np, "sensor-data", &sensor_id)) { dev_info(&pdev->dev, - "'sensor-id' missing in the node '%s'\n", - np->name); + "'sensor-id' missing in the node '%pOFn'\n", + np); continue; } diff --git a/drivers/hwmon/iio_hwmon.c b/drivers/hwmon/iio_hwmon.c index 2f3f875c06ac..eed66e533ee2 100644 --- a/drivers/hwmon/iio_hwmon.c +++ b/drivers/hwmon/iio_hwmon.c @@ -65,13 +65,9 @@ static int iio_hwmon_probe(struct platform_device *pdev) int in_i = 1, temp_i = 1, curr_i = 1, humidity_i = 1; enum iio_chan_type type; struct iio_channel *channels; - const char *name = "iio_hwmon"; struct device *hwmon_dev; char *sname; - if (dev->of_node && dev->of_node->name) - name = dev->of_node->name; - channels = devm_iio_channel_get_all(dev); if (IS_ERR(channels)) { if (PTR_ERR(channels) == -ENODEV) @@ -141,11 +137,15 @@ static int iio_hwmon_probe(struct platform_device *pdev) st->attr_group.attrs = st->attrs; st->groups[0] = &st->attr_group; - sname = devm_kstrdup(dev, name, GFP_KERNEL); - if (!sname) - return -ENOMEM; + if (dev->of_node) { + sname = devm_kasprintf(dev, GFP_KERNEL, "%pOFn", dev->of_node); + if (!sname) + return -ENOMEM; + strreplace(sname, '-', '_'); + } else { + sname = "iio_hwmon"; + } - strreplace(sname, '-', '_'); hwmon_dev = devm_hwmon_device_register_with_groups(dev, sname, st, st->groups); return PTR_ERR_OR_ZERO(hwmon_dev); diff --git a/drivers/hwmon/ina3221.c b/drivers/hwmon/ina3221.c index e6b49500c52a..d61688f04594 100644 --- a/drivers/hwmon/ina3221.c +++ b/drivers/hwmon/ina3221.c @@ -38,9 +38,12 @@ #define INA3221_WARN3 0x0c #define INA3221_MASK_ENABLE 0x0f -#define INA3221_CONFIG_MODE_SHUNT BIT(1) -#define INA3221_CONFIG_MODE_BUS BIT(2) -#define INA3221_CONFIG_MODE_CONTINUOUS BIT(3) +#define INA3221_CONFIG_MODE_MASK GENMASK(2, 0) +#define INA3221_CONFIG_MODE_POWERDOWN 0 +#define INA3221_CONFIG_MODE_SHUNT BIT(0) +#define INA3221_CONFIG_MODE_BUS BIT(1) +#define INA3221_CONFIG_MODE_CONTINUOUS BIT(2) +#define INA3221_CONFIG_CHx_EN(x) BIT(14 - (x)) #define INA3221_RSHUNT_DEFAULT 10000 @@ -74,30 +77,37 @@ enum ina3221_channels { INA3221_NUM_CHANNELS }; -static const unsigned int register_channel[] = { - [INA3221_SHUNT1] = INA3221_CHANNEL1, - [INA3221_SHUNT2] = INA3221_CHANNEL2, - [INA3221_SHUNT3] = INA3221_CHANNEL3, - [INA3221_CRIT1] = INA3221_CHANNEL1, - [INA3221_CRIT2] = INA3221_CHANNEL2, - [INA3221_CRIT3] = INA3221_CHANNEL3, - [INA3221_WARN1] = INA3221_CHANNEL1, - [INA3221_WARN2] = INA3221_CHANNEL2, - [INA3221_WARN3] = INA3221_CHANNEL3, +/** + * struct ina3221_input - channel input source specific information + * @label: label of channel input source + * @shunt_resistor: shunt resistor value of channel input source + * @disconnected: connection status of channel input source + */ +struct ina3221_input { + const char *label; + int shunt_resistor; + bool disconnected; }; /** * struct ina3221_data - device specific information * @regmap: Register map of the device * @fields: Register fields of the device - * @shunt_resistors: Array of resistor values per channel + * @inputs: Array of channel input source specific structures + * @reg_config: Register value of INA3221_CONFIG */ struct ina3221_data { struct regmap *regmap; struct regmap_field *fields[F_MAX_FIELDS]; - int shunt_resistors[INA3221_NUM_CHANNELS]; + struct ina3221_input inputs[INA3221_NUM_CHANNELS]; + u32 reg_config; }; +static inline bool ina3221_is_enabled(struct ina3221_data *ina, int channel) +{ + return ina->reg_config & INA3221_CONFIG_CHx_EN(channel); +} + static int ina3221_read_value(struct ina3221_data *ina, unsigned int reg, int *val) { @@ -113,107 +123,284 @@ static int ina3221_read_value(struct ina3221_data *ina, unsigned int reg, return 0; } -static ssize_t ina3221_show_bus_voltage(struct device *dev, - struct device_attribute *attr, - char *buf) +static const u8 ina3221_in_reg[] = { + INA3221_BUS1, + INA3221_BUS2, + INA3221_BUS3, + INA3221_SHUNT1, + INA3221_SHUNT2, + INA3221_SHUNT3, +}; + +static int ina3221_read_in(struct device *dev, u32 attr, int channel, long *val) { - struct sensor_device_attribute *sd_attr = to_sensor_dev_attr(attr); + const bool is_shunt = channel > INA3221_CHANNEL3; struct ina3221_data *ina = dev_get_drvdata(dev); - unsigned int reg = sd_attr->index; - int val, voltage_mv, ret; - - ret = ina3221_read_value(ina, reg, &val); - if (ret) - return ret; + u8 reg = ina3221_in_reg[channel]; + int regval, ret; + + /* Translate shunt channel index to sensor channel index */ + channel %= INA3221_NUM_CHANNELS; + + switch (attr) { + case hwmon_in_input: + if (!ina3221_is_enabled(ina, channel)) + return -ENODATA; + + ret = ina3221_read_value(ina, reg, ®val); + if (ret) + return ret; + + /* + * Scale of shunt voltage (uV): LSB is 40uV + * Scale of bus voltage (mV): LSB is 8mV + */ + *val = regval * (is_shunt ? 40 : 8); + return 0; + case hwmon_in_enable: + *val = ina3221_is_enabled(ina, channel); + return 0; + default: + return -EOPNOTSUPP; + } +} - voltage_mv = val * 8; +static const u8 ina3221_curr_reg[][INA3221_NUM_CHANNELS] = { + [hwmon_curr_input] = { INA3221_SHUNT1, INA3221_SHUNT2, INA3221_SHUNT3 }, + [hwmon_curr_max] = { INA3221_WARN1, INA3221_WARN2, INA3221_WARN3 }, + [hwmon_curr_crit] = { INA3221_CRIT1, INA3221_CRIT2, INA3221_CRIT3 }, + [hwmon_curr_max_alarm] = { F_WF1, F_WF2, F_WF3 }, + [hwmon_curr_crit_alarm] = { F_CF1, F_CF2, F_CF3 }, +}; - return snprintf(buf, PAGE_SIZE, "%d\n", voltage_mv); +static int ina3221_read_curr(struct device *dev, u32 attr, + int channel, long *val) +{ + struct ina3221_data *ina = dev_get_drvdata(dev); + struct ina3221_input *input = &ina->inputs[channel]; + int resistance_uo = input->shunt_resistor; + u8 reg = ina3221_curr_reg[attr][channel]; + int regval, voltage_nv, ret; + + switch (attr) { + case hwmon_curr_input: + if (!ina3221_is_enabled(ina, channel)) + return -ENODATA; + /* fall through */ + case hwmon_curr_crit: + case hwmon_curr_max: + ret = ina3221_read_value(ina, reg, ®val); + if (ret) + return ret; + + /* Scale of shunt voltage: LSB is 40uV (40000nV) */ + voltage_nv = regval * 40000; + /* Return current in mA */ + *val = DIV_ROUND_CLOSEST(voltage_nv, resistance_uo); + return 0; + case hwmon_curr_crit_alarm: + case hwmon_curr_max_alarm: + ret = regmap_field_read(ina->fields[reg], ®val); + if (ret) + return ret; + *val = regval; + return 0; + default: + return -EOPNOTSUPP; + } } -static ssize_t ina3221_show_shunt_voltage(struct device *dev, - struct device_attribute *attr, - char *buf) +static int ina3221_write_curr(struct device *dev, u32 attr, + int channel, long val) { - struct sensor_device_attribute *sd_attr = to_sensor_dev_attr(attr); struct ina3221_data *ina = dev_get_drvdata(dev); - unsigned int reg = sd_attr->index; - int val, voltage_uv, ret; + struct ina3221_input *input = &ina->inputs[channel]; + int resistance_uo = input->shunt_resistor; + u8 reg = ina3221_curr_reg[attr][channel]; + int regval, current_ma, voltage_uv; - ret = ina3221_read_value(ina, reg, &val); - if (ret) - return ret; - voltage_uv = val * 40; + /* clamp current */ + current_ma = clamp_val(val, + INT_MIN / resistance_uo, + INT_MAX / resistance_uo); + + voltage_uv = DIV_ROUND_CLOSEST(current_ma * resistance_uo, 1000); - return snprintf(buf, PAGE_SIZE, "%d\n", voltage_uv); + /* clamp voltage */ + voltage_uv = clamp_val(voltage_uv, -163800, 163800); + + /* 1 / 40uV(scale) << 3(register shift) = 5 */ + regval = DIV_ROUND_CLOSEST(voltage_uv, 5) & 0xfff8; + + return regmap_write(ina->regmap, reg, regval); } -static ssize_t ina3221_show_current(struct device *dev, - struct device_attribute *attr, char *buf) +static int ina3221_write_enable(struct device *dev, int channel, bool enable) { - struct sensor_device_attribute *sd_attr = to_sensor_dev_attr(attr); struct ina3221_data *ina = dev_get_drvdata(dev); - unsigned int reg = sd_attr->index; - unsigned int channel = register_channel[reg]; - int resistance_uo = ina->shunt_resistors[channel]; - int val, current_ma, voltage_nv, ret; + u16 config, mask = INA3221_CONFIG_CHx_EN(channel); + int ret; - ret = ina3221_read_value(ina, reg, &val); + config = enable ? mask : 0; + + /* Enable or disable the channel */ + ret = regmap_update_bits(ina->regmap, INA3221_CONFIG, mask, config); if (ret) return ret; - voltage_nv = val * 40000; - current_ma = DIV_ROUND_CLOSEST(voltage_nv, resistance_uo); + /* Cache the latest config register value */ + ret = regmap_read(ina->regmap, INA3221_CONFIG, &ina->reg_config); + if (ret) + return ret; - return snprintf(buf, PAGE_SIZE, "%d\n", current_ma); + return 0; } -static ssize_t ina3221_set_current(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) +static int ina3221_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + switch (type) { + case hwmon_in: + /* 0-align channel ID */ + return ina3221_read_in(dev, attr, channel - 1, val); + case hwmon_curr: + return ina3221_read_curr(dev, attr, channel, val); + default: + return -EOPNOTSUPP; + } +} + +static int ina3221_write(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long val) +{ + switch (type) { + case hwmon_in: + /* 0-align channel ID */ + return ina3221_write_enable(dev, channel - 1, val); + case hwmon_curr: + return ina3221_write_curr(dev, attr, channel, val); + default: + return -EOPNOTSUPP; + } +} + +static int ina3221_read_string(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, const char **str) { - struct sensor_device_attribute *sd_attr = to_sensor_dev_attr(attr); struct ina3221_data *ina = dev_get_drvdata(dev); - unsigned int reg = sd_attr->index; - unsigned int channel = register_channel[reg]; - int resistance_uo = ina->shunt_resistors[channel]; - int val, current_ma, voltage_uv, ret; + int index = channel - 1; - ret = kstrtoint(buf, 0, ¤t_ma); - if (ret) - return ret; + *str = ina->inputs[index].label; - /* clamp current */ - current_ma = clamp_val(current_ma, - INT_MIN / resistance_uo, - INT_MAX / resistance_uo); + return 0; +} - voltage_uv = DIV_ROUND_CLOSEST(current_ma * resistance_uo, 1000); +static umode_t ina3221_is_visible(const void *drvdata, + enum hwmon_sensor_types type, + u32 attr, int channel) +{ + const struct ina3221_data *ina = drvdata; + const struct ina3221_input *input = NULL; + + switch (type) { + case hwmon_in: + /* Ignore in0_ */ + if (channel == 0) + return 0; + + switch (attr) { + case hwmon_in_label: + if (channel - 1 <= INA3221_CHANNEL3) + input = &ina->inputs[channel - 1]; + /* Hide label node if label is not provided */ + return (input && input->label) ? 0444 : 0; + case hwmon_in_input: + return 0444; + case hwmon_in_enable: + return 0644; + default: + return 0; + } + case hwmon_curr: + switch (attr) { + case hwmon_curr_input: + case hwmon_curr_crit_alarm: + case hwmon_curr_max_alarm: + return 0444; + case hwmon_curr_crit: + case hwmon_curr_max: + return 0644; + default: + return 0; + } + default: + return 0; + } +} - /* clamp voltage */ - voltage_uv = clamp_val(voltage_uv, -163800, 163800); +static const u32 ina3221_in_config[] = { + /* 0: dummy, skipped in is_visible */ + HWMON_I_INPUT, + /* 1-3: input voltage Channels */ + HWMON_I_INPUT | HWMON_I_ENABLE | HWMON_I_LABEL, + HWMON_I_INPUT | HWMON_I_ENABLE | HWMON_I_LABEL, + HWMON_I_INPUT | HWMON_I_ENABLE | HWMON_I_LABEL, + /* 4-6: shunt voltage Channels */ + HWMON_I_INPUT, + HWMON_I_INPUT, + HWMON_I_INPUT, + 0 +}; - /* 1 / 40uV(scale) << 3(register shift) = 5 */ - val = DIV_ROUND_CLOSEST(voltage_uv, 5) & 0xfff8; +static const struct hwmon_channel_info ina3221_in = { + .type = hwmon_in, + .config = ina3221_in_config, +}; - ret = regmap_write(ina->regmap, reg, val); - if (ret) - return ret; +#define INA3221_HWMON_CURR_CONFIG (HWMON_C_INPUT | \ + HWMON_C_CRIT | HWMON_C_CRIT_ALARM | \ + HWMON_C_MAX | HWMON_C_MAX_ALARM) - return count; -} +static const u32 ina3221_curr_config[] = { + INA3221_HWMON_CURR_CONFIG, + INA3221_HWMON_CURR_CONFIG, + INA3221_HWMON_CURR_CONFIG, + 0 +}; + +static const struct hwmon_channel_info ina3221_curr = { + .type = hwmon_curr, + .config = ina3221_curr_config, +}; + +static const struct hwmon_channel_info *ina3221_info[] = { + &ina3221_in, + &ina3221_curr, + NULL +}; + +static const struct hwmon_ops ina3221_hwmon_ops = { + .is_visible = ina3221_is_visible, + .read_string = ina3221_read_string, + .read = ina3221_read, + .write = ina3221_write, +}; +static const struct hwmon_chip_info ina3221_chip_info = { + .ops = &ina3221_hwmon_ops, + .info = ina3221_info, +}; + +/* Extra attribute groups */ static ssize_t ina3221_show_shunt(struct device *dev, struct device_attribute *attr, char *buf) { struct sensor_device_attribute *sd_attr = to_sensor_dev_attr(attr); struct ina3221_data *ina = dev_get_drvdata(dev); unsigned int channel = sd_attr->index; - unsigned int resistance_uo; - - resistance_uo = ina->shunt_resistors[channel]; + struct ina3221_input *input = &ina->inputs[channel]; - return snprintf(buf, PAGE_SIZE, "%d\n", resistance_uo); + return snprintf(buf, PAGE_SIZE, "%d\n", input->shunt_resistor); } static ssize_t ina3221_set_shunt(struct device *dev, @@ -223,6 +410,7 @@ static ssize_t ina3221_set_shunt(struct device *dev, struct sensor_device_attribute *sd_attr = to_sensor_dev_attr(attr); struct ina3221_data *ina = dev_get_drvdata(dev); unsigned int channel = sd_attr->index; + struct ina3221_input *input = &ina->inputs[channel]; int val; int ret; @@ -232,43 +420,11 @@ static ssize_t ina3221_set_shunt(struct device *dev, val = clamp_val(val, 1, INT_MAX); - ina->shunt_resistors[channel] = val; + input->shunt_resistor = val; return count; } -static ssize_t ina3221_show_alert(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct sensor_device_attribute *sd_attr = to_sensor_dev_attr(attr); - struct ina3221_data *ina = dev_get_drvdata(dev); - unsigned int field = sd_attr->index; - unsigned int regval; - int ret; - - ret = regmap_field_read(ina->fields[field], ®val); - if (ret) - return ret; - - return snprintf(buf, PAGE_SIZE, "%d\n", regval); -} - -/* bus voltage */ -static SENSOR_DEVICE_ATTR(in1_input, S_IRUGO, - ina3221_show_bus_voltage, NULL, INA3221_BUS1); -static SENSOR_DEVICE_ATTR(in2_input, S_IRUGO, - ina3221_show_bus_voltage, NULL, INA3221_BUS2); -static SENSOR_DEVICE_ATTR(in3_input, S_IRUGO, - ina3221_show_bus_voltage, NULL, INA3221_BUS3); - -/* calculated current */ -static SENSOR_DEVICE_ATTR(curr1_input, S_IRUGO, - ina3221_show_current, NULL, INA3221_SHUNT1); -static SENSOR_DEVICE_ATTR(curr2_input, S_IRUGO, - ina3221_show_current, NULL, INA3221_SHUNT2); -static SENSOR_DEVICE_ATTR(curr3_input, S_IRUGO, - ina3221_show_current, NULL, INA3221_SHUNT3); - /* shunt resistance */ static SENSOR_DEVICE_ATTR(shunt1_resistor, S_IRUGO | S_IWUSR, ina3221_show_shunt, ina3221_set_shunt, INA3221_CHANNEL1); @@ -277,83 +433,16 @@ static SENSOR_DEVICE_ATTR(shunt2_resistor, S_IRUGO | S_IWUSR, static SENSOR_DEVICE_ATTR(shunt3_resistor, S_IRUGO | S_IWUSR, ina3221_show_shunt, ina3221_set_shunt, INA3221_CHANNEL3); -/* critical current */ -static SENSOR_DEVICE_ATTR(curr1_crit, S_IRUGO | S_IWUSR, - ina3221_show_current, ina3221_set_current, INA3221_CRIT1); -static SENSOR_DEVICE_ATTR(curr2_crit, S_IRUGO | S_IWUSR, - ina3221_show_current, ina3221_set_current, INA3221_CRIT2); -static SENSOR_DEVICE_ATTR(curr3_crit, S_IRUGO | S_IWUSR, - ina3221_show_current, ina3221_set_current, INA3221_CRIT3); - -/* critical current alert */ -static SENSOR_DEVICE_ATTR(curr1_crit_alarm, S_IRUGO, - ina3221_show_alert, NULL, F_CF1); -static SENSOR_DEVICE_ATTR(curr2_crit_alarm, S_IRUGO, - ina3221_show_alert, NULL, F_CF2); -static SENSOR_DEVICE_ATTR(curr3_crit_alarm, S_IRUGO, - ina3221_show_alert, NULL, F_CF3); - -/* warning current */ -static SENSOR_DEVICE_ATTR(curr1_max, S_IRUGO | S_IWUSR, - ina3221_show_current, ina3221_set_current, INA3221_WARN1); -static SENSOR_DEVICE_ATTR(curr2_max, S_IRUGO | S_IWUSR, - ina3221_show_current, ina3221_set_current, INA3221_WARN2); -static SENSOR_DEVICE_ATTR(curr3_max, S_IRUGO | S_IWUSR, - ina3221_show_current, ina3221_set_current, INA3221_WARN3); - -/* warning current alert */ -static SENSOR_DEVICE_ATTR(curr1_max_alarm, S_IRUGO, - ina3221_show_alert, NULL, F_WF1); -static SENSOR_DEVICE_ATTR(curr2_max_alarm, S_IRUGO, - ina3221_show_alert, NULL, F_WF2); -static SENSOR_DEVICE_ATTR(curr3_max_alarm, S_IRUGO, - ina3221_show_alert, NULL, F_WF3); - -/* shunt voltage */ -static SENSOR_DEVICE_ATTR(in4_input, S_IRUGO, - ina3221_show_shunt_voltage, NULL, INA3221_SHUNT1); -static SENSOR_DEVICE_ATTR(in5_input, S_IRUGO, - ina3221_show_shunt_voltage, NULL, INA3221_SHUNT2); -static SENSOR_DEVICE_ATTR(in6_input, S_IRUGO, - ina3221_show_shunt_voltage, NULL, INA3221_SHUNT3); - static struct attribute *ina3221_attrs[] = { - /* channel 1 */ - &sensor_dev_attr_in1_input.dev_attr.attr, - &sensor_dev_attr_curr1_input.dev_attr.attr, &sensor_dev_attr_shunt1_resistor.dev_attr.attr, - &sensor_dev_attr_curr1_crit.dev_attr.attr, - &sensor_dev_attr_curr1_crit_alarm.dev_attr.attr, - &sensor_dev_attr_curr1_max.dev_attr.attr, - &sensor_dev_attr_curr1_max_alarm.dev_attr.attr, - &sensor_dev_attr_in4_input.dev_attr.attr, - - /* channel 2 */ - &sensor_dev_attr_in2_input.dev_attr.attr, - &sensor_dev_attr_curr2_input.dev_attr.attr, &sensor_dev_attr_shunt2_resistor.dev_attr.attr, - &sensor_dev_attr_curr2_crit.dev_attr.attr, - &sensor_dev_attr_curr2_crit_alarm.dev_attr.attr, - &sensor_dev_attr_curr2_max.dev_attr.attr, - &sensor_dev_attr_curr2_max_alarm.dev_attr.attr, - &sensor_dev_attr_in5_input.dev_attr.attr, - - /* channel 3 */ - &sensor_dev_attr_in3_input.dev_attr.attr, - &sensor_dev_attr_curr3_input.dev_attr.attr, &sensor_dev_attr_shunt3_resistor.dev_attr.attr, - &sensor_dev_attr_curr3_crit.dev_attr.attr, - &sensor_dev_attr_curr3_crit_alarm.dev_attr.attr, - &sensor_dev_attr_curr3_max.dev_attr.attr, - &sensor_dev_attr_curr3_max_alarm.dev_attr.attr, - &sensor_dev_attr_in6_input.dev_attr.attr, - NULL, }; ATTRIBUTE_GROUPS(ina3221); static const struct regmap_range ina3221_yes_ranges[] = { - regmap_reg_range(INA3221_SHUNT1, INA3221_BUS3), + regmap_reg_range(INA3221_CONFIG, INA3221_BUS3), regmap_reg_range(INA3221_MASK_ENABLE, INA3221_MASK_ENABLE), }; @@ -370,6 +459,66 @@ static const struct regmap_config ina3221_regmap_config = { .volatile_table = &ina3221_volatile_table, }; +static int ina3221_probe_child_from_dt(struct device *dev, + struct device_node *child, + struct ina3221_data *ina) +{ + struct ina3221_input *input; + u32 val; + int ret; + + ret = of_property_read_u32(child, "reg", &val); + if (ret) { + dev_err(dev, "missing reg property of %s\n", child->name); + return ret; + } else if (val > INA3221_CHANNEL3) { + dev_err(dev, "invalid reg %d of %s\n", val, child->name); + return ret; + } + + input = &ina->inputs[val]; + + /* Log the disconnected channel input */ + if (!of_device_is_available(child)) { + input->disconnected = true; + return 0; + } + + /* Save the connected input label if available */ + of_property_read_string(child, "label", &input->label); + + /* Overwrite default shunt resistor value optionally */ + if (!of_property_read_u32(child, "shunt-resistor-micro-ohms", &val)) { + if (val < 1 || val > INT_MAX) { + dev_err(dev, "invalid shunt resistor value %u of %s\n", + val, child->name); + return -EINVAL; + } + input->shunt_resistor = val; + } + + return 0; +} + +static int ina3221_probe_from_dt(struct device *dev, struct ina3221_data *ina) +{ + const struct device_node *np = dev->of_node; + struct device_node *child; + int ret; + + /* Compatible with non-DT platforms */ + if (!np) + return 0; + + for_each_child_of_node(np, child) { + ret = ina3221_probe_child_from_dt(dev, child, ina); + if (ret) + return ret; + } + + return 0; +} + static int ina3221_probe(struct i2c_client *client, const struct i2c_device_id *id) { @@ -399,7 +548,13 @@ static int ina3221_probe(struct i2c_client *client, } for (i = 0; i < INA3221_NUM_CHANNELS; i++) - ina->shunt_resistors[i] = INA3221_RSHUNT_DEFAULT; + ina->inputs[i].shunt_resistor = INA3221_RSHUNT_DEFAULT; + + ret = ina3221_probe_from_dt(dev, ina); + if (ret) { + dev_err(dev, "Unable to probe from device tree\n"); + return ret; + } ret = regmap_field_write(ina->fields[F_RST], true); if (ret) { @@ -407,9 +562,25 @@ static int ina3221_probe(struct i2c_client *client, return ret; } - hwmon_dev = devm_hwmon_device_register_with_groups(dev, - client->name, - ina, ina3221_groups); + /* Sync config register after reset */ + ret = regmap_read(ina->regmap, INA3221_CONFIG, &ina->reg_config); + if (ret) + return ret; + + /* Disable channels if their inputs are disconnected */ + for (i = 0; i < INA3221_NUM_CHANNELS; i++) { + if (ina->inputs[i].disconnected) + ina->reg_config &= ~INA3221_CONFIG_CHx_EN(i); + } + ret = regmap_write(ina->regmap, INA3221_CONFIG, ina->reg_config); + if (ret) + return ret; + + dev_set_drvdata(dev, ina); + + hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name, ina, + &ina3221_chip_info, + ina3221_groups); if (IS_ERR(hwmon_dev)) { dev_err(dev, "Unable to register hwmon device\n"); return PTR_ERR(hwmon_dev); @@ -418,6 +589,60 @@ static int ina3221_probe(struct i2c_client *client, return 0; } +static int __maybe_unused ina3221_suspend(struct device *dev) +{ + struct ina3221_data *ina = dev_get_drvdata(dev); + int ret; + + /* Save config register value and enable cache-only */ + ret = regmap_read(ina->regmap, INA3221_CONFIG, &ina->reg_config); + if (ret) + return ret; + + /* Set to power-down mode for power saving */ + ret = regmap_update_bits(ina->regmap, INA3221_CONFIG, + INA3221_CONFIG_MODE_MASK, + INA3221_CONFIG_MODE_POWERDOWN); + if (ret) + return ret; + + regcache_cache_only(ina->regmap, true); + regcache_mark_dirty(ina->regmap); + + return 0; +} + +static int __maybe_unused ina3221_resume(struct device *dev) +{ + struct ina3221_data *ina = dev_get_drvdata(dev); + int ret; + + regcache_cache_only(ina->regmap, false); + + /* Software reset the chip */ + ret = regmap_field_write(ina->fields[F_RST], true); + if (ret) { + dev_err(dev, "Unable to reset device\n"); + return ret; + } + + /* Restore cached register values to hardware */ + ret = regcache_sync(ina->regmap); + if (ret) + return ret; + + /* Restore config register value to hardware */ + ret = regmap_write(ina->regmap, INA3221_CONFIG, ina->reg_config); + if (ret) + return ret; + + return 0; +} + +static const struct dev_pm_ops ina3221_pm = { + SET_SYSTEM_SLEEP_PM_OPS(ina3221_suspend, ina3221_resume) +}; + static const struct of_device_id ina3221_of_match_table[] = { { .compatible = "ti,ina3221", }, { /* sentinel */ } @@ -435,6 +660,7 @@ static struct i2c_driver ina3221_i2c_driver = { .driver = { .name = INA3221_DRIVER_NAME, .of_match_table = ina3221_of_match_table, + .pm = &ina3221_pm, }, .id_table = ina3221_ids, }; diff --git a/drivers/hwmon/k10temp.c b/drivers/hwmon/k10temp.c index bb15d7816a29..2cef0c37ff6f 100644 --- a/drivers/hwmon/k10temp.c +++ b/drivers/hwmon/k10temp.c @@ -325,8 +325,9 @@ static int k10temp_probe(struct pci_dev *pdev, data->pdev = pdev; - if (boot_cpu_data.x86 == 0x15 && (boot_cpu_data.x86_model == 0x60 || - boot_cpu_data.x86_model == 0x70)) { + if (boot_cpu_data.x86 == 0x15 && + ((boot_cpu_data.x86_model & 0xf0) == 0x60 || + (boot_cpu_data.x86_model & 0xf0) == 0x70)) { data->read_htcreg = read_htcreg_nb_f15; data->read_tempreg = read_tempreg_nb_f15; } else if (boot_cpu_data.x86 == 0x17) { diff --git a/drivers/hwmon/lm75.c b/drivers/hwmon/lm75.c index 49f4b33a5685..c7f20543b2bf 100644 --- a/drivers/hwmon/lm75.c +++ b/drivers/hwmon/lm75.c @@ -47,6 +47,7 @@ enum lm75_type { /* keep sorted in alphabetical order */ lm75b, max6625, max6626, + max31725, mcp980x, stds75, tcn75, @@ -64,7 +65,6 @@ enum lm75_type { /* keep sorted in alphabetical order */ static const unsigned short normal_i2c[] = { 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, I2C_CLIENT_END }; - /* The LM75 registers */ #define LM75_REG_TEMP 0x00 #define LM75_REG_CONF 0x01 @@ -76,7 +76,7 @@ struct lm75_data { struct i2c_client *client; struct regmap *regmap; u8 orig_conf; - u8 resolution; /* In bits, between 9 and 12 */ + u8 resolution; /* In bits, between 9 and 16 */ u8 resolution_limits; unsigned int sample_time; /* In ms */ }; @@ -254,7 +254,8 @@ static const struct regmap_config lm75_regmap_config = { .volatile_reg = lm75_is_volatile_reg, .val_format_endian = REGMAP_ENDIAN_BIG, .cache_type = REGCACHE_RBTREE, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; static void lm75_remove(void *data) @@ -339,6 +340,10 @@ lm75_probe(struct i2c_client *client, const struct i2c_device_id *id) data->resolution_limits = 9; data->sample_time = MSEC_PER_SEC / 4; break; + case max31725: + data->resolution = 16; + data->sample_time = MSEC_PER_SEC / 8; + break; case tcn75: data->resolution = 9; data->sample_time = MSEC_PER_SEC / 8; @@ -415,6 +420,8 @@ static const struct i2c_device_id lm75_ids[] = { { "lm75b", lm75b, }, { "max6625", max6625, }, { "max6626", max6626, }, + { "max31725", max31725, }, + { "max31726", max31725, }, { "mcp980x", mcp980x, }, { "stds75", stds75, }, { "tcn75", tcn75, }, @@ -472,6 +479,14 @@ static const struct of_device_id lm75_of_match[] = { .data = (void *)max6626 }, { + .compatible = "maxim,max31725", + .data = (void *)max31725 + }, + { + .compatible = "maxim,max31726", + .data = (void *)max31725 + }, + { .compatible = "maxim,mcp980x", .data = (void *)mcp980x }, diff --git a/drivers/hwmon/lm92.c b/drivers/hwmon/lm92.c index d40fe5122e94..e7333f8e185c 100644 --- a/drivers/hwmon/lm92.c +++ b/drivers/hwmon/lm92.c @@ -127,8 +127,8 @@ static struct lm92_data *lm92_update_device(struct device *dev) mutex_lock(&data->update_lock); - if (time_after(jiffies, data->last_updated + HZ) - || !data->valid) { + if (time_after(jiffies, data->last_updated + HZ) || + !data->valid) { dev_dbg(&client->dev, "Updating lm92 data\n"); for (i = 0; i < t_num_regs; i++) { data->temp[i] = @@ -153,7 +153,7 @@ static ssize_t show_temp(struct device *dev, struct device_attribute *devattr, } static ssize_t set_temp(struct device *dev, struct device_attribute *devattr, - const char *buf, size_t count) + const char *buf, size_t count) { struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); struct lm92_data *data = dev_get_drvdata(dev); @@ -161,7 +161,7 @@ static ssize_t set_temp(struct device *dev, struct device_attribute *devattr, int nr = attr->index; long val; int err; - + err = kstrtol(buf, 10, &val); if (err) return err; @@ -178,6 +178,7 @@ static ssize_t show_temp_hyst(struct device *dev, { struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); struct lm92_data *data = lm92_update_device(dev); + return sprintf(buf, "%d\n", TEMP_FROM_REG(data->temp[attr->index]) - TEMP_FROM_REG(data->temp[t_hyst])); } @@ -186,6 +187,7 @@ static ssize_t temp1_min_hyst_show(struct device *dev, struct device_attribute *attr, char *buf) { struct lm92_data *data = lm92_update_device(dev); + return sprintf(buf, "%d\n", TEMP_FROM_REG(data->temp[t_min]) + TEMP_FROM_REG(data->temp[t_hyst])); } @@ -206,7 +208,7 @@ static ssize_t set_temp_hyst(struct device *dev, val = clamp_val(val, -120000, 220000); mutex_lock(&data->update_lock); - data->temp[t_hyst] = + data->temp[t_hyst] = TEMP_TO_REG(TEMP_FROM_REG(data->temp[attr->index]) - val); i2c_smbus_write_word_swapped(client, LM92_REG_TEMP_HYST, data->temp[t_hyst]); @@ -218,6 +220,7 @@ static ssize_t alarms_show(struct device *dev, struct device_attribute *attr, char *buf) { struct lm92_data *data = lm92_update_device(dev); + return sprintf(buf, "%d\n", ALARMS_FROM_REG(data->temp[t_input])); } @@ -324,7 +327,6 @@ static int lm92_probe(struct i2c_client *new_client, return PTR_ERR_OR_ZERO(hwmon_dev); } - /* * Module and driver stuff */ diff --git a/drivers/hwmon/lm95245.c b/drivers/hwmon/lm95245.c index 27cb06d65594..996b50246175 100644 --- a/drivers/hwmon/lm95245.c +++ b/drivers/hwmon/lm95245.c @@ -541,7 +541,8 @@ static const struct regmap_config lm95245_regmap_config = { .writeable_reg = lm95245_is_writeable_reg, .volatile_reg = lm95245_is_volatile_reg, .cache_type = REGCACHE_RBTREE, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; static const u32 lm95245_chip_config[] = { diff --git a/drivers/hwmon/mc13783-adc.c b/drivers/hwmon/mc13783-adc.c index 78fe8759d2a9..825b922a3f92 100644 --- a/drivers/hwmon/mc13783-adc.c +++ b/drivers/hwmon/mc13783-adc.c @@ -1,21 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * Driver for the ADC on Freescale Semiconductor MC13783 and MC13892 PMICs. * * Copyright 2004-2007 Freescale Semiconductor, Inc. All Rights Reserved. * Copyright (C) 2009 Sascha Hauer, Pengutronix - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 51 - * Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include <linux/mfd/mc13xxx.h> diff --git a/drivers/hwmon/nct6775.c b/drivers/hwmon/nct6775.c index 78603b78cf41..c3040079b1cb 100644 --- a/drivers/hwmon/nct6775.c +++ b/drivers/hwmon/nct6775.c @@ -42,6 +42,10 @@ * nct6793d 15 6 6 2+6 0xd120 0xc1 0x5ca3 * nct6795d 14 6 6 2+6 0xd350 0xc1 0x5ca3 * nct6796d 14 7 7 2+6 0xd420 0xc1 0x5ca3 + * nct6797d 14 7 7 2+6 0xd450 0xc1 0x5ca3 + * (0xd451) + * nct6798d 14 7 7 2+6 0xd458 0xc1 0x5ca3 + * (0xd459) * * #temp lists the number of monitored temperature sources (first value) plus * the number of directly connectable temperature sensors (second value). @@ -69,7 +73,7 @@ #define USE_ALTERNATE enum kinds { nct6106, nct6775, nct6776, nct6779, nct6791, nct6792, nct6793, - nct6795, nct6796 }; + nct6795, nct6796, nct6797, nct6798 }; /* used to set data->name = nct6775_device_names[data->sio_kind] */ static const char * const nct6775_device_names[] = { @@ -82,6 +86,8 @@ static const char * const nct6775_device_names[] = { "nct6793", "nct6795", "nct6796", + "nct6797", + "nct6798", }; static const char * const nct6775_sio_names[] __initconst = { @@ -94,6 +100,8 @@ static const char * const nct6775_sio_names[] __initconst = { "NCT6793D", "NCT6795D", "NCT6796D", + "NCT6797D", + "NCT6798D", }; static unsigned short force_id; @@ -129,7 +137,9 @@ MODULE_PARM_DESC(fan_debounce, "Enable debouncing for fan RPM signal"); #define SIO_NCT6793_ID 0xd120 #define SIO_NCT6795_ID 0xd350 #define SIO_NCT6796_ID 0xd420 -#define SIO_ID_MASK 0xFFF0 +#define SIO_NCT6797_ID 0xd450 +#define SIO_NCT6798_ID 0xd458 +#define SIO_ID_MASK 0xFFF8 enum pwm_enable { off, manual, thermal_cruise, speed_cruise, sf3, sf4 }; @@ -504,7 +514,7 @@ static const s8 NCT6779_BEEP_BITS[] = { static const u16 NCT6779_REG_FAN[] = { 0x4c0, 0x4c2, 0x4c4, 0x4c6, 0x4c8, 0x4ca, 0x4ce }; static const u16 NCT6779_REG_FAN_PULSES[NUM_FAN] = { - 0x644, 0x645, 0x646, 0x647, 0x648, 0x649 }; + 0x644, 0x645, 0x646, 0x647, 0x648, 0x649, 0x64f }; static const u16 NCT6779_REG_CRITICAL_PWM_ENABLE[] = { 0x136, 0x236, 0x336, 0x836, 0x936, 0xa36, 0xb36 }; @@ -704,10 +714,10 @@ static const char *const nct6795_temp_label[] = { "PCH_CHIP_TEMP", "PCH_CPU_TEMP", "PCH_MCH_TEMP", - "PCH_DIM0_TEMP", - "PCH_DIM1_TEMP", - "PCH_DIM2_TEMP", - "PCH_DIM3_TEMP", + "Agent0 Dimm0", + "Agent0 Dimm1", + "Agent1 Dimm0", + "Agent1 Dimm1", "BYTE_TEMP0", "BYTE_TEMP1", "PECI Agent 0 Calibration", @@ -742,10 +752,10 @@ static const char *const nct6796_temp_label[] = { "PCH_CHIP_TEMP", "PCH_CPU_TEMP", "PCH_MCH_TEMP", - "PCH_DIM0_TEMP", - "PCH_DIM1_TEMP", - "PCH_DIM2_TEMP", - "PCH_DIM3_TEMP", + "Agent0 Dimm0", + "Agent0 Dimm1", + "Agent1 Dimm0", + "Agent1 Dimm1", "BYTE_TEMP0", "BYTE_TEMP1", "PECI Agent 0 Calibration", @@ -757,6 +767,44 @@ static const char *const nct6796_temp_label[] = { #define NCT6796_TEMP_MASK 0xbfff0ffe #define NCT6796_VIRT_TEMP_MASK 0x80000c00 +static const char *const nct6798_temp_label[] = { + "", + "SYSTIN", + "CPUTIN", + "AUXTIN0", + "AUXTIN1", + "AUXTIN2", + "AUXTIN3", + "AUXTIN4", + "SMBUSMASTER 0", + "SMBUSMASTER 1", + "Virtual_TEMP", + "Virtual_TEMP", + "", + "", + "", + "", + "PECI Agent 0", + "PECI Agent 1", + "PCH_CHIP_CPU_MAX_TEMP", + "PCH_CHIP_TEMP", + "PCH_CPU_TEMP", + "PCH_MCH_TEMP", + "Agent0 Dimm0", + "Agent0 Dimm1", + "Agent1 Dimm0", + "Agent1 Dimm1", + "BYTE_TEMP0", + "BYTE_TEMP1", + "", + "", + "", + "Virtual_TEMP" +}; + +#define NCT6798_TEMP_MASK 0x8fff0ffe +#define NCT6798_VIRT_TEMP_MASK 0x80000c00 + /* NCT6102D/NCT6106D specific data */ #define NCT6106_REG_VBAT 0x318 @@ -1288,6 +1336,8 @@ static bool is_word_sized(struct nct6775_data *data, u16 reg) case nct6793: case nct6795: case nct6796: + case nct6797: + case nct6798: return reg == 0x150 || reg == 0x153 || reg == 0x155 || (reg & 0xfff0) == 0x4c0 || reg == 0x402 || @@ -1643,6 +1693,8 @@ static void nct6775_update_pwm_limits(struct device *dev) case nct6793: case nct6795: case nct6796: + case nct6797: + case nct6798: reg = nct6775_read_value(data, data->REG_CRITICAL_PWM_ENABLE[i]); if (reg & data->CRITICAL_PWM_ENABLE_MASK) @@ -2847,6 +2899,8 @@ store_temp_tolerance(struct device *dev, struct device_attribute *attr, * Fan speed tolerance is a tricky beast, since the associated register is * a tick counter, but the value is reported and configured as rpm. * Compute resulting low and high rpm values and report the difference. + * A fan speed tolerance only makes sense if a fan target speed has been + * configured, so only display values other than 0 if that is the case. */ static ssize_t show_speed_tolerance(struct device *dev, struct device_attribute *attr, @@ -2855,19 +2909,23 @@ show_speed_tolerance(struct device *dev, struct device_attribute *attr, struct nct6775_data *data = nct6775_update_device(dev); struct sensor_device_attribute *sattr = to_sensor_dev_attr(attr); int nr = sattr->index; - int low = data->target_speed[nr] - data->target_speed_tolerance[nr]; - int high = data->target_speed[nr] + data->target_speed_tolerance[nr]; - int tolerance; - - if (low <= 0) - low = 1; - if (high > 0xffff) - high = 0xffff; - if (high < low) - high = low; - - tolerance = (fan_from_reg16(low, data->fan_div[nr]) - - fan_from_reg16(high, data->fan_div[nr])) / 2; + int target = data->target_speed[nr]; + int tolerance = 0; + + if (target) { + int low = target - data->target_speed_tolerance[nr]; + int high = target + data->target_speed_tolerance[nr]; + + if (low <= 0) + low = 1; + if (high > 0xffff) + high = 0xffff; + if (high < low) + high = low; + + tolerance = (fan_from_reg16(low, data->fan_div[nr]) + - fan_from_reg16(high, data->fan_div[nr])) / 2; + } return sprintf(buf, "%d\n", tolerance); } @@ -3071,6 +3129,8 @@ store_auto_pwm(struct device *dev, struct device_attribute *attr, case nct6793: case nct6795: case nct6796: + case nct6797: + case nct6798: nct6775_write_value(data, data->REG_CRITICAL_PWM[nr], val); reg = nct6775_read_value(data, @@ -3430,7 +3490,6 @@ nct6775_check_fan_inputs(struct nct6775_data *data) bool pwm3pin = false, pwm4pin = false, pwm5pin = false; bool pwm6pin = false, pwm7pin = false; int sioreg = data->sioreg; - int regval; /* Store SIO_REG_ENABLE for use during resume */ superio_select(sioreg, NCT6775_LD_HWM); @@ -3438,10 +3497,10 @@ nct6775_check_fan_inputs(struct nct6775_data *data) /* fan4 and fan5 share some pins with the GPIO and serial flash */ if (data->kind == nct6775) { - regval = superio_inb(sioreg, 0x2c); + int cr2c = superio_inb(sioreg, 0x2c); - fan3pin = regval & BIT(6); - pwm3pin = regval & BIT(7); + fan3pin = cr2c & BIT(6); + pwm3pin = cr2c & BIT(7); /* On NCT6775, fan4 shares pins with the fdc interface */ fan4pin = !(superio_inb(sioreg, 0x2A) & 0x80); @@ -3486,85 +3545,130 @@ nct6775_check_fan_inputs(struct nct6775_data *data) fan4min = fan4pin; pwm3pin = fan3pin; } else if (data->kind == nct6106) { - regval = superio_inb(sioreg, 0x24); - fan3pin = !(regval & 0x80); - pwm3pin = regval & 0x08; - } else { - /* NCT6779D, NCT6791D, NCT6792D, NCT6793D, NCT6795D, NCT6796D */ - int regval_1b, regval_2a, regval_2f; - bool dsw_en; - - regval = superio_inb(sioreg, 0x1c); + int cr24 = superio_inb(sioreg, 0x24); - fan3pin = !(regval & BIT(5)); - fan4pin = !(regval & BIT(6)); - fan5pin = !(regval & BIT(7)); - - pwm3pin = !(regval & BIT(0)); - pwm4pin = !(regval & BIT(1)); - pwm5pin = !(regval & BIT(2)); + fan3pin = !(cr24 & 0x80); + pwm3pin = cr24 & 0x08; + } else { + /* + * NCT6779D, NCT6791D, NCT6792D, NCT6793D, NCT6795D, NCT6796D, + * NCT6797D, NCT6798D + */ + int cr1a = superio_inb(sioreg, 0x1a); + int cr1b = superio_inb(sioreg, 0x1b); + int cr1c = superio_inb(sioreg, 0x1c); + int cr1d = superio_inb(sioreg, 0x1d); + int cr2a = superio_inb(sioreg, 0x2a); + int cr2b = superio_inb(sioreg, 0x2b); + int cr2d = superio_inb(sioreg, 0x2d); + int cr2f = superio_inb(sioreg, 0x2f); + bool dsw_en = cr2f & BIT(3); + bool ddr4_en = cr2f & BIT(4); + int cre0; + int creb; + int cred; + + superio_select(sioreg, NCT6775_LD_12); + cre0 = superio_inb(sioreg, 0xe0); + creb = superio_inb(sioreg, 0xeb); + cred = superio_inb(sioreg, 0xed); + + fan3pin = !(cr1c & BIT(5)); + fan4pin = !(cr1c & BIT(6)); + fan5pin = !(cr1c & BIT(7)); + + pwm3pin = !(cr1c & BIT(0)); + pwm4pin = !(cr1c & BIT(1)); + pwm5pin = !(cr1c & BIT(2)); - regval = superio_inb(sioreg, 0x2d); switch (data->kind) { case nct6791: + fan6pin = cr2d & BIT(1); + pwm6pin = cr2d & BIT(0); + break; case nct6792: - fan6pin = regval & BIT(1); - pwm6pin = regval & BIT(0); + fan6pin = !dsw_en && (cr2d & BIT(1)); + pwm6pin = !dsw_en && (cr2d & BIT(0)); break; case nct6793: + fan5pin |= cr1b & BIT(5); + fan5pin |= creb & BIT(5); + + fan6pin = creb & BIT(3); + + pwm5pin |= cr2d & BIT(7); + pwm5pin |= (creb & BIT(4)) && !(cr2a & BIT(0)); + + pwm6pin = !dsw_en && (cr2d & BIT(0)); + pwm6pin |= creb & BIT(2); + break; case nct6795: + fan5pin |= cr1b & BIT(5); + fan5pin |= creb & BIT(5); + + fan6pin = (cr2a & BIT(4)) && + (!dsw_en || (cred & BIT(4))); + fan6pin |= creb & BIT(3); + + pwm5pin |= cr2d & BIT(7); + pwm5pin |= (creb & BIT(4)) && !(cr2a & BIT(0)); + + pwm6pin = (cr2a & BIT(3)) && (cred & BIT(2)); + pwm6pin |= creb & BIT(2); + break; case nct6796: - regval_1b = superio_inb(sioreg, 0x1b); - regval_2a = superio_inb(sioreg, 0x2a); - regval_2f = superio_inb(sioreg, 0x2f); - dsw_en = regval_2f & BIT(3); + fan5pin |= cr1b & BIT(5); + fan5pin |= (cre0 & BIT(3)) && !(cr1b & BIT(0)); + fan5pin |= creb & BIT(5); - if (!pwm5pin) - pwm5pin = regval & BIT(7); + fan6pin = (cr2a & BIT(4)) && + (!dsw_en || (cred & BIT(4))); + fan6pin |= creb & BIT(3); - if (!fan5pin) - fan5pin = regval_1b & BIT(5); + fan7pin = !(cr2b & BIT(2)); - superio_select(sioreg, NCT6775_LD_12); - if (data->kind != nct6796) { - int regval_eb = superio_inb(sioreg, 0xeb); + pwm5pin |= cr2d & BIT(7); + pwm5pin |= (cre0 & BIT(4)) && !(cr1b & BIT(0)); + pwm5pin |= (creb & BIT(4)) && !(cr2a & BIT(0)); - if (!dsw_en) { - fan6pin = regval & BIT(1); - pwm6pin = regval & BIT(0); - } + pwm6pin = (cr2a & BIT(3)) && (cred & BIT(2)); + pwm6pin |= creb & BIT(2); - if (!fan5pin) - fan5pin = regval_eb & BIT(5); - if (!pwm5pin) - pwm5pin = (regval_eb & BIT(4)) && - !(regval_2a & BIT(0)); - if (!fan6pin) - fan6pin = regval_eb & BIT(3); - if (!pwm6pin) - pwm6pin = regval_eb & BIT(2); - } + pwm7pin = !(cr1d & (BIT(2) | BIT(3))); + break; + case nct6797: + fan5pin |= !ddr4_en && (cr1b & BIT(5)); + fan5pin |= creb & BIT(5); - if (data->kind == nct6795 || data->kind == nct6796) { - int regval_ed = superio_inb(sioreg, 0xed); + fan6pin = cr2a & BIT(4); + fan6pin |= creb & BIT(3); - if (!fan6pin) - fan6pin = (regval_2a & BIT(4)) && - (!dsw_en || - (dsw_en && (regval_ed & BIT(4)))); - if (!pwm6pin) - pwm6pin = (regval_2a & BIT(3)) && - (regval_ed & BIT(2)); - } + fan7pin = cr1a & BIT(1); - if (data->kind == nct6796) { - int regval_1d = superio_inb(sioreg, 0x1d); - int regval_2b = superio_inb(sioreg, 0x2b); + pwm5pin |= (creb & BIT(4)) && !(cr2a & BIT(0)); + pwm5pin |= !ddr4_en && (cr2d & BIT(7)); - fan7pin = !(regval_2b & BIT(2)); - pwm7pin = !(regval_1d & (BIT(2) | BIT(3))); - } + pwm6pin = creb & BIT(2); + pwm6pin |= cred & BIT(2); + pwm7pin = cr1d & BIT(4); + break; + case nct6798: + fan6pin = !(cr1b & BIT(0)) && (cre0 & BIT(3)); + fan6pin |= cr2a & BIT(4); + fan6pin |= creb & BIT(5); + + fan7pin = cr1b & BIT(5); + fan7pin |= !(cr2b & BIT(2)); + fan7pin |= creb & BIT(3); + + pwm6pin = !(cr1b & BIT(0)) && (cre0 & BIT(4)); + pwm6pin |= !(cred & BIT(2)) && (cr2a & BIT(3)); + pwm6pin |= (creb & BIT(4)) && !(cr2a & BIT(0)); + + pwm7pin = !(cr1d & (BIT(2) | BIT(3))); + pwm7pin |= cr2d & BIT(7); + pwm7pin |= creb & BIT(2); break; default: /* NCT6779D */ break; @@ -3943,8 +4047,12 @@ static int nct6775_probe(struct platform_device *pdev) case nct6793: case nct6795: case nct6796: + case nct6797: + case nct6798: data->in_num = 15; - data->pwm_num = (data->kind == nct6796) ? 7 : 6; + data->pwm_num = (data->kind == nct6796 || + data->kind == nct6797 || + data->kind == nct6798) ? 7 : 6; data->auto_pwm_num = 4; data->has_fan_div = false; data->temp_fixed_num = 6; @@ -3978,6 +4086,7 @@ static int nct6775_probe(struct platform_device *pdev) data->virt_temp_mask = NCT6793_VIRT_TEMP_MASK; break; case nct6795: + case nct6797: data->temp_label = nct6795_temp_label; data->temp_mask = NCT6795_TEMP_MASK; data->virt_temp_mask = NCT6795_VIRT_TEMP_MASK; @@ -3987,6 +4096,11 @@ static int nct6775_probe(struct platform_device *pdev) data->temp_mask = NCT6796_TEMP_MASK; data->virt_temp_mask = NCT6796_VIRT_TEMP_MASK; break; + case nct6798: + data->temp_label = nct6798_temp_label; + data->temp_mask = NCT6798_TEMP_MASK; + data->virt_temp_mask = NCT6798_VIRT_TEMP_MASK; + break; } data->REG_CONFIG = NCT6775_REG_CONFIG; @@ -4256,6 +4370,8 @@ static int nct6775_probe(struct platform_device *pdev) case nct6793: case nct6795: case nct6796: + case nct6797: + case nct6798: break; } @@ -4291,6 +4407,8 @@ static int nct6775_probe(struct platform_device *pdev) case nct6793: case nct6795: case nct6796: + case nct6797: + case nct6798: tmp |= 0x7e; break; } @@ -4493,6 +4611,12 @@ static int __init nct6775_find(int sioaddr, struct nct6775_sio_data *sio_data) case SIO_NCT6796_ID: sio_data->kind = nct6796; break; + case SIO_NCT6797_ID: + sio_data->kind = nct6797; + break; + case SIO_NCT6798_ID: + sio_data->kind = nct6798; + break; default: if (val != 0xffff) pr_debug("unsupported chip ID: 0x%04x\n", val); diff --git a/drivers/hwmon/npcm750-pwm-fan.c b/drivers/hwmon/npcm750-pwm-fan.c index b998f9fbed41..b3b907bdfb63 100644 --- a/drivers/hwmon/npcm750-pwm-fan.c +++ b/drivers/hwmon/npcm750-pwm-fan.c @@ -52,7 +52,7 @@ /* Define the Counter Register, value = 100 for match 100% */ #define NPCM7XX_PWM_COUNTER_DEFAULT_NUM 255 -#define NPCM7XX_PWM_CMR_DEFAULT_NUM 127 +#define NPCM7XX_PWM_CMR_DEFAULT_NUM 255 #define NPCM7XX_PWM_CMR_MAX 255 /* default all PWM channels PRESCALE2 = 1 */ @@ -861,7 +861,7 @@ static int npcm7xx_create_pwm_cooling(struct device *dev, dev_err(dev, "Property 'cooling-levels' cannot be read.\n"); return ret; } - snprintf(cdev->name, THERMAL_NAME_LENGTH, "%s%d", child->name, + snprintf(cdev->name, THERMAL_NAME_LENGTH, "%pOFn%d", child, pwm_port); cdev->tcdev = thermal_of_cooling_device_register(child, diff --git a/drivers/hwmon/pmbus/Kconfig b/drivers/hwmon/pmbus/Kconfig index a82018aaf473..629cb45f8557 100644 --- a/drivers/hwmon/pmbus/Kconfig +++ b/drivers/hwmon/pmbus/Kconfig @@ -5,7 +5,6 @@ menuconfig PMBUS tristate "PMBus support" depends on I2C - default n help Say yes here if you want to enable PMBus support. @@ -28,7 +27,6 @@ config SENSORS_PMBUS config SENSORS_ADM1275 tristate "Analog Devices ADM1275 and compatibles" - default n help If you say yes here you get hardware monitoring support for Analog Devices ADM1075, ADM1272, ADM1275, ADM1276, ADM1278, ADM1293, @@ -49,7 +47,6 @@ config SENSORS_IBM_CFFPS config SENSORS_IR35221 tristate "Infineon IR35221" - default n help If you say yes here you get hardware monitoring support for the Infineon IR35221 controller. @@ -59,7 +56,6 @@ config SENSORS_IR35221 config SENSORS_LM25066 tristate "National Semiconductor LM25066 and compatibles" - default n help If you say yes here you get hardware monitoring support for National Semiconductor LM25056, LM25066, LM5064, and LM5066. @@ -69,7 +65,6 @@ config SENSORS_LM25066 config SENSORS_LTC2978 tristate "Linear Technologies LTC2978 and compatibles" - default n help If you say yes here you get hardware monitoring support for Linear Technology LTC2974, LTC2975, LTC2977, LTC2978, LTC2980, LTC3880, @@ -83,11 +78,11 @@ config SENSORS_LTC2978_REGULATOR depends on SENSORS_LTC2978 && REGULATOR help If you say yes here you get regulator support for Linear - Technology LTC2974, LTC2977, LTC2978, LTC3880, LTC3883, and LTM4676. + Technology LTC2974, LTC2977, LTC2978, LTC3880, LTC3883, LTM4676 + and LTM4686. config SENSORS_LTC3815 tristate "Linear Technologies LTC3815" - default n help If you say yes here you get hardware monitoring support for Linear Technology LTC3815. @@ -97,7 +92,6 @@ config SENSORS_LTC3815 config SENSORS_MAX16064 tristate "Maxim MAX16064" - default n help If you say yes here you get hardware monitoring support for Maxim MAX16064. @@ -107,7 +101,6 @@ config SENSORS_MAX16064 config SENSORS_MAX20751 tristate "Maxim MAX20751" - default n help If you say yes here you get hardware monitoring support for Maxim MAX20751. @@ -117,7 +110,6 @@ config SENSORS_MAX20751 config SENSORS_MAX31785 tristate "Maxim MAX31785 and compatibles" - default n help If you say yes here you get hardware monitoring support for Maxim MAX31785. @@ -127,7 +119,6 @@ config SENSORS_MAX31785 config SENSORS_MAX34440 tristate "Maxim MAX34440 and compatibles" - default n help If you say yes here you get hardware monitoring support for Maxim MAX34440, MAX34441, MAX34446, MAX34451, MAX34460, and MAX34461. @@ -137,7 +128,6 @@ config SENSORS_MAX34440 config SENSORS_MAX8688 tristate "Maxim MAX8688" - default n help If you say yes here you get hardware monitoring support for Maxim MAX8688. @@ -147,7 +137,6 @@ config SENSORS_MAX8688 config SENSORS_TPS40422 tristate "TI TPS40422" - default n help If you say yes here you get hardware monitoring support for TI TPS40422. @@ -166,7 +155,6 @@ config SENSORS_TPS53679 config SENSORS_UCD9000 tristate "TI UCD90120, UCD90124, UCD90160, UCD9090, UCD90910" - default n help If you say yes here you get hardware monitoring support for TI UCD90120, UCD90124, UCD90160, UCD9090, UCD90910, Sequencer and System @@ -177,7 +165,6 @@ config SENSORS_UCD9000 config SENSORS_UCD9200 tristate "TI UCD9220, UCD9222, UCD9224, UCD9240, UCD9244, UCD9246, UCD9248" - default n help If you say yes here you get hardware monitoring support for TI UCD9220, UCD9222, UCD9224, UCD9240, UCD9244, UCD9246, and UCD9248 @@ -188,7 +175,6 @@ config SENSORS_UCD9200 config SENSORS_ZL6100 tristate "Intersil ZL6100 and compatibles" - default n help If you say yes here you get hardware monitoring support for Intersil ZL2004, ZL2005, ZL2006, ZL2008, ZL2105, ZL2106, ZL6100, ZL6105, diff --git a/drivers/hwmon/pmbus/ltc2978.c b/drivers/hwmon/pmbus/ltc2978.c index 58b789c28b48..07afb92bb36b 100644 --- a/drivers/hwmon/pmbus/ltc2978.c +++ b/drivers/hwmon/pmbus/ltc2978.c @@ -4,6 +4,7 @@ * Copyright (c) 2011 Ericsson AB. * Copyright (c) 2013, 2014, 2015 Guenter Roeck * Copyright (c) 2015 Linear Technology + * Copyright (c) 2018 Analog Devices Inc. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -28,7 +29,7 @@ #include "pmbus.h" enum chips { ltc2974, ltc2975, ltc2977, ltc2978, ltc2980, ltc3880, ltc3882, - ltc3883, ltc3886, ltc3887, ltm2987, ltm4675, ltm4676 }; + ltc3883, ltc3886, ltc3887, ltm2987, ltm4675, ltm4676, ltm4686 }; /* Common for all chips */ #define LTC2978_MFR_VOUT_PEAK 0xdd @@ -81,6 +82,7 @@ enum chips { ltc2974, ltc2975, ltc2977, ltc2978, ltc2980, ltc3880, ltc3882, #define LTM4676_ID_REV1 0x4400 #define LTM4676_ID_REV2 0x4480 #define LTM4676A_ID 0x47e0 +#define LTM4686_ID 0x4770 #define LTC2974_NUM_PAGES 4 #define LTC2978_NUM_PAGES 8 @@ -512,6 +514,7 @@ static const struct i2c_device_id ltc2978_id[] = { {"ltm2987", ltm2987}, {"ltm4675", ltm4675}, {"ltm4676", ltm4676}, + {"ltm4686", ltm4686}, {} }; MODULE_DEVICE_TABLE(i2c, ltc2978_id); @@ -588,6 +591,8 @@ static int ltc2978_get_id(struct i2c_client *client) else if (chip_id == LTM4676_ID_REV1 || chip_id == LTM4676_ID_REV2 || chip_id == LTM4676A_ID) return ltm4676; + else if (chip_id == LTM4686_ID) + return ltm4686; dev_err(&client->dev, "Unsupported chip ID 0x%x\n", chip_id); return -ENODEV; @@ -684,6 +689,7 @@ static int ltc2978_probe(struct i2c_client *client, case ltc3887: case ltm4675: case ltm4676: + case ltm4686: data->features |= FEAT_CLEAR_PEAKS | FEAT_NEEDS_POLLING; info->read_word_data = ltc3880_read_word_data; info->pages = LTC3880_NUM_PAGES; @@ -770,6 +776,7 @@ static const struct of_device_id ltc2978_of_match[] = { { .compatible = "lltc,ltm2987" }, { .compatible = "lltc,ltm4675" }, { .compatible = "lltc,ltm4676" }, + { .compatible = "lltc,ltm4686" }, { } }; MODULE_DEVICE_TABLE(of, ltc2978_of_match); diff --git a/drivers/hwmon/pmbus/pmbus.c b/drivers/hwmon/pmbus/pmbus.c index 7718e58dbda5..7688dab32f6e 100644 --- a/drivers/hwmon/pmbus/pmbus.c +++ b/drivers/hwmon/pmbus/pmbus.c @@ -118,6 +118,8 @@ static int pmbus_identify(struct i2c_client *client, } else { info->pages = 1; } + + pmbus_clear_faults(client); } if (pmbus_check_byte_register(client, 0, PMBUS_VOUT_MODE)) { diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c index 82c3754e21e3..2e2b5851139c 100644 --- a/drivers/hwmon/pmbus/pmbus_core.c +++ b/drivers/hwmon/pmbus/pmbus_core.c @@ -2015,7 +2015,10 @@ static int pmbus_init_common(struct i2c_client *client, struct pmbus_data *data, if (ret >= 0 && (ret & PB_CAPABILITY_ERROR_CHECK)) client->flags |= I2C_CLIENT_PEC; - pmbus_clear_faults(client); + if (data->info->pages) + pmbus_clear_faults(client); + else + pmbus_clear_fault_page(client, -1); if (info->identify) { ret = (*info->identify)(client, info); diff --git a/drivers/hwmon/pwm-fan.c b/drivers/hwmon/pwm-fan.c index 7838af58f92d..7da6a160d45a 100644 --- a/drivers/hwmon/pwm-fan.c +++ b/drivers/hwmon/pwm-fan.c @@ -221,8 +221,12 @@ static int pwm_fan_probe(struct platform_device *pdev) ctx->pwm = devm_of_pwm_get(&pdev->dev, pdev->dev.of_node, NULL); if (IS_ERR(ctx->pwm)) { - dev_err(&pdev->dev, "Could not get PWM\n"); - return PTR_ERR(ctx->pwm); + ret = PTR_ERR(ctx->pwm); + + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "Could not get PWM: %d\n", ret); + + return ret; } platform_set_drvdata(pdev, ctx); @@ -290,9 +294,19 @@ static int pwm_fan_remove(struct platform_device *pdev) static int pwm_fan_suspend(struct device *dev) { struct pwm_fan_ctx *ctx = dev_get_drvdata(dev); + struct pwm_args args; + int ret; + + pwm_get_args(ctx->pwm, &args); + + if (ctx->pwm_value) { + ret = pwm_config(ctx->pwm, 0, args.period); + if (ret < 0) + return ret; - if (ctx->pwm_value) pwm_disable(ctx->pwm); + } + return 0; } diff --git a/drivers/hwmon/scmi-hwmon.c b/drivers/hwmon/scmi-hwmon.c index 91976b6ca300..2e005edee0c9 100644 --- a/drivers/hwmon/scmi-hwmon.c +++ b/drivers/hwmon/scmi-hwmon.c @@ -56,7 +56,7 @@ scmi_hwmon_is_visible(const void *drvdata, enum hwmon_sensor_types type, const struct scmi_sensors *scmi_sensors = drvdata; sensor = *(scmi_sensors->info[type] + channel); - if (sensor && sensor->name) + if (sensor) return S_IRUGO; return 0; diff --git a/drivers/hwmon/scpi-hwmon.c b/drivers/hwmon/scpi-hwmon.c index 7e49da50bc69..111d521e2189 100644 --- a/drivers/hwmon/scpi-hwmon.c +++ b/drivers/hwmon/scpi-hwmon.c @@ -286,10 +286,8 @@ static int scpi_hwmon_probe(struct platform_device *pdev) * any thermal zones or if the thermal subsystem is * not configured. */ - if (IS_ERR(z)) { + if (IS_ERR(z)) devm_kfree(dev, zone); - continue; - } } return 0; diff --git a/drivers/hwmon/sht15.c b/drivers/hwmon/sht15.c index 2be77752cd56..c878242f3486 100644 --- a/drivers/hwmon/sht15.c +++ b/drivers/hwmon/sht15.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * sht15.c - support for the SHT15 Temperature and Humidity Sensor * @@ -9,10 +10,6 @@ * * Copyright (c) 2007 Wouter Horre * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * * For further information, see the Documentation/hwmon/sht15 file. */ diff --git a/drivers/hwmon/tmp102.c b/drivers/hwmon/tmp102.c index dfc40c740d07..6778283e36f9 100644 --- a/drivers/hwmon/tmp102.c +++ b/drivers/hwmon/tmp102.c @@ -212,7 +212,8 @@ static const struct regmap_config tmp102_regmap_config = { .volatile_reg = tmp102_is_volatile_reg, .val_format_endian = REGMAP_ENDIAN_BIG, .cache_type = REGCACHE_RBTREE, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; static int tmp102_probe(struct i2c_client *client, diff --git a/drivers/hwmon/tmp108.c b/drivers/hwmon/tmp108.c index 91bb94639286..429bfeae4ca8 100644 --- a/drivers/hwmon/tmp108.c +++ b/drivers/hwmon/tmp108.c @@ -345,7 +345,8 @@ static const struct regmap_config tmp108_regmap_config = { .volatile_reg = tmp108_is_volatile_reg, .val_format_endian = REGMAP_ENDIAN_BIG, .cache_type = REGCACHE_RBTREE, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; static int tmp108_probe(struct i2c_client *client, diff --git a/drivers/hwmon/tmp421.c b/drivers/hwmon/tmp421.c index e36399213324..8844c9565d2a 100644 --- a/drivers/hwmon/tmp421.c +++ b/drivers/hwmon/tmp421.c @@ -226,8 +226,10 @@ static int tmp421_detect(struct i2c_client *client, { enum chips kind; struct i2c_adapter *adapter = client->adapter; - const char * const names[] = { "TMP421", "TMP422", "TMP423", - "TMP441", "TMP442" }; + static const char * const names[] = { + "TMP421", "TMP422", "TMP423", + "TMP441", "TMP442" + }; int addr = client->addr; u8 reg; diff --git a/drivers/i2c/busses/i2c-rcar.c b/drivers/i2c/busses/i2c-rcar.c index 52cf42b32f0a..4aa7dde876f3 100644 --- a/drivers/i2c/busses/i2c-rcar.c +++ b/drivers/i2c/busses/i2c-rcar.c @@ -806,8 +806,12 @@ static int rcar_i2c_master_xfer(struct i2c_adapter *adap, time_left = wait_event_timeout(priv->wait, priv->flags & ID_DONE, num * adap->timeout); - if (!time_left) { + + /* cleanup DMA if it couldn't complete properly due to an error */ + if (priv->dma_direction != DMA_NONE) rcar_i2c_cleanup_dma(priv); + + if (!time_left) { rcar_i2c_init(priv); ret = -ETIMEDOUT; } else if (priv->flags & ID_NACK) { diff --git a/drivers/ide/ide-cd.c b/drivers/ide/ide-cd.c index 44a7a255ef74..f9b59d41813f 100644 --- a/drivers/ide/ide-cd.c +++ b/drivers/ide/ide-cd.c @@ -1784,7 +1784,7 @@ static int ide_cd_probe(ide_drive_t *drive) ide_cd_read_toc(drive); g->fops = &idecd_ops; g->flags |= GENHD_FL_REMOVABLE | GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE; - device_add_disk(&drive->gendev, g); + device_add_disk(&drive->gendev, g, NULL); return 0; out_free_disk: diff --git a/drivers/ide/ide-gd.c b/drivers/ide/ide-gd.c index e823394ed543..04e008e8f6f9 100644 --- a/drivers/ide/ide-gd.c +++ b/drivers/ide/ide-gd.c @@ -416,7 +416,7 @@ static int ide_gd_probe(ide_drive_t *drive) if (drive->dev_flags & IDE_DFLAG_REMOVABLE) g->flags = GENHD_FL_REMOVABLE; g->fops = &ide_gd_ops; - device_add_disk(&drive->gendev, g); + device_add_disk(&drive->gendev, g, NULL); return 0; out_free_disk: diff --git a/drivers/iio/light/apds9960.c b/drivers/iio/light/apds9960.c index 1f112ae15f3c..b09b8b60bd83 100644 --- a/drivers/iio/light/apds9960.c +++ b/drivers/iio/light/apds9960.c @@ -206,7 +206,8 @@ static const struct regmap_config apds9960_regmap_config = { .name = APDS9960_REGMAP_NAME, .reg_bits = 8, .val_bits = 8, - .use_single_rw = 1, + .use_single_read = true, + .use_single_write = true, .volatile_table = &apds9960_volatile_table, .precious_table = &apds9960_precious_table, diff --git a/drivers/iio/light/max44000.c b/drivers/iio/light/max44000.c index bcdb0eb9e537..4067dff2ff6a 100644 --- a/drivers/iio/light/max44000.c +++ b/drivers/iio/light/max44000.c @@ -473,17 +473,18 @@ static bool max44000_precious_reg(struct device *dev, unsigned int reg) } static const struct regmap_config max44000_regmap_config = { - .reg_bits = 8, - .val_bits = 8, - - .max_register = MAX44000_REG_PRX_DATA, - .readable_reg = max44000_readable_reg, - .writeable_reg = max44000_writeable_reg, - .volatile_reg = max44000_volatile_reg, - .precious_reg = max44000_precious_reg, - - .use_single_rw = 1, - .cache_type = REGCACHE_RBTREE, + .reg_bits = 8, + .val_bits = 8, + + .max_register = MAX44000_REG_PRX_DATA, + .readable_reg = max44000_readable_reg, + .writeable_reg = max44000_writeable_reg, + .volatile_reg = max44000_volatile_reg, + .precious_reg = max44000_precious_reg, + + .use_single_read = true, + .use_single_write = true, + .cache_type = REGCACHE_RBTREE, }; static irqreturn_t max44000_trigger_handler(int irq, void *p) diff --git a/drivers/iio/temperature/mlx90632.c b/drivers/iio/temperature/mlx90632.c index 9851311aa3fd..be03be719efe 100644 --- a/drivers/iio/temperature/mlx90632.c +++ b/drivers/iio/temperature/mlx90632.c @@ -140,7 +140,8 @@ static const struct regmap_config mlx90632_regmap = { .rd_table = &mlx90632_readable_regs_tbl, .wr_table = &mlx90632_writeable_regs_tbl, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, .reg_format_endian = REGMAP_ENDIAN_BIG, .val_format_endian = REGMAP_ENDIAN_BIG, .cache_type = REGCACHE_RBTREE, diff --git a/drivers/infiniband/core/ucm.c b/drivers/infiniband/core/ucm.c index faa9e6116b2f..73332b9a25b5 100644 --- a/drivers/infiniband/core/ucm.c +++ b/drivers/infiniband/core/ucm.c @@ -46,6 +46,8 @@ #include <linux/mutex.h> #include <linux/slab.h> +#include <linux/nospec.h> + #include <linux/uaccess.h> #include <rdma/ib.h> @@ -1120,6 +1122,7 @@ static ssize_t ib_ucm_write(struct file *filp, const char __user *buf, if (hdr.cmd >= ARRAY_SIZE(ucm_cmd_table)) return -EINVAL; + hdr.cmd = array_index_nospec(hdr.cmd, ARRAY_SIZE(ucm_cmd_table)); if (hdr.in + sizeof(hdr) > len) return -EINVAL; diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c index 21863ddde63e..01d68ed46c1b 100644 --- a/drivers/infiniband/core/ucma.c +++ b/drivers/infiniband/core/ucma.c @@ -44,6 +44,8 @@ #include <linux/module.h> #include <linux/nsproxy.h> +#include <linux/nospec.h> + #include <rdma/rdma_user_cm.h> #include <rdma/ib_marshall.h> #include <rdma/rdma_cm.h> @@ -1676,6 +1678,7 @@ static ssize_t ucma_write(struct file *filp, const char __user *buf, if (hdr.cmd >= ARRAY_SIZE(ucma_cmd_table)) return -EINVAL; + hdr.cmd = array_index_nospec(hdr.cmd, ARRAY_SIZE(ucma_cmd_table)); if (hdr.in + sizeof(hdr) > len) return -EINVAL; diff --git a/drivers/input/mouse/elan_i2c_core.c b/drivers/input/mouse/elan_i2c_core.c index f5ae24865355..b0f9d19b3410 100644 --- a/drivers/input/mouse/elan_i2c_core.c +++ b/drivers/input/mouse/elan_i2c_core.c @@ -1346,6 +1346,7 @@ static const struct acpi_device_id elan_acpi_id[] = { { "ELAN0611", 0 }, { "ELAN0612", 0 }, { "ELAN0618", 0 }, + { "ELAN061C", 0 }, { "ELAN061D", 0 }, { "ELAN0622", 0 }, { "ELAN1000", 0 }, diff --git a/drivers/input/touchscreen/tsc200x-core.c b/drivers/input/touchscreen/tsc200x-core.c index e0fde590df8e..62973ac01381 100644 --- a/drivers/input/touchscreen/tsc200x-core.c +++ b/drivers/input/touchscreen/tsc200x-core.c @@ -68,7 +68,8 @@ const struct regmap_config tsc200x_regmap_config = { .read_flag_mask = TSC200X_REG_READ, .write_flag_mask = TSC200X_REG_PND0, .wr_table = &tsc200x_writable_table, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; EXPORT_SYMBOL_GPL(tsc200x_regmap_config); diff --git a/drivers/lightnvm/Kconfig b/drivers/lightnvm/Kconfig index 439bf90d084d..a872cd720967 100644 --- a/drivers/lightnvm/Kconfig +++ b/drivers/lightnvm/Kconfig @@ -4,8 +4,7 @@ menuconfig NVM bool "Open-Channel SSD target support" - depends on BLOCK && PCI - select BLK_DEV_NVME + depends on BLOCK help Say Y here to get to enable Open-channel SSDs. diff --git a/drivers/lightnvm/core.c b/drivers/lightnvm/core.c index 60aa7bc5a630..efb976a863d2 100644 --- a/drivers/lightnvm/core.c +++ b/drivers/lightnvm/core.c @@ -355,6 +355,11 @@ static int nvm_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create) return -EINVAL; } + if ((tt->flags & NVM_TGT_F_HOST_L2P) != (dev->geo.dom & NVM_RSP_L2P)) { + pr_err("nvm: device is incompatible with target L2P type.\n"); + return -EINVAL; + } + if (nvm_target_exists(create->tgtname)) { pr_err("nvm: target name already exists (%s)\n", create->tgtname); @@ -598,22 +603,16 @@ static void nvm_ppa_dev_to_tgt(struct nvm_tgt_dev *tgt_dev, static void nvm_rq_tgt_to_dev(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd) { - if (rqd->nr_ppas == 1) { - nvm_ppa_tgt_to_dev(tgt_dev, &rqd->ppa_addr, 1); - return; - } + struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); - nvm_ppa_tgt_to_dev(tgt_dev, rqd->ppa_list, rqd->nr_ppas); + nvm_ppa_tgt_to_dev(tgt_dev, ppa_list, rqd->nr_ppas); } static void nvm_rq_dev_to_tgt(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd) { - if (rqd->nr_ppas == 1) { - nvm_ppa_dev_to_tgt(tgt_dev, &rqd->ppa_addr, 1); - return; - } + struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); - nvm_ppa_dev_to_tgt(tgt_dev, rqd->ppa_list, rqd->nr_ppas); + nvm_ppa_dev_to_tgt(tgt_dev, ppa_list, rqd->nr_ppas); } int nvm_register_tgt_type(struct nvm_tgt_type *tt) @@ -712,45 +711,23 @@ static void nvm_free_rqd_ppalist(struct nvm_tgt_dev *tgt_dev, nvm_dev_dma_free(tgt_dev->parent, rqd->ppa_list, rqd->dma_ppa_list); } -int nvm_get_chunk_meta(struct nvm_tgt_dev *tgt_dev, struct nvm_chk_meta *meta, - struct ppa_addr ppa, int nchks) +static int nvm_set_flags(struct nvm_geo *geo, struct nvm_rq *rqd) { - struct nvm_dev *dev = tgt_dev->parent; - - nvm_ppa_tgt_to_dev(tgt_dev, &ppa, 1); - - return dev->ops->get_chk_meta(tgt_dev->parent, meta, - (sector_t)ppa.ppa, nchks); -} -EXPORT_SYMBOL(nvm_get_chunk_meta); - -int nvm_set_tgt_bb_tbl(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *ppas, - int nr_ppas, int type) -{ - struct nvm_dev *dev = tgt_dev->parent; - struct nvm_rq rqd; - int ret; + int flags = 0; - if (nr_ppas > NVM_MAX_VLBA) { - pr_err("nvm: unable to update all blocks atomically\n"); - return -EINVAL; - } + if (geo->version == NVM_OCSSD_SPEC_20) + return 0; - memset(&rqd, 0, sizeof(struct nvm_rq)); + if (rqd->is_seq) + flags |= geo->pln_mode >> 1; - nvm_set_rqd_ppalist(tgt_dev, &rqd, ppas, nr_ppas); - nvm_rq_tgt_to_dev(tgt_dev, &rqd); + if (rqd->opcode == NVM_OP_PREAD) + flags |= (NVM_IO_SCRAMBLE_ENABLE | NVM_IO_SUSPEND); + else if (rqd->opcode == NVM_OP_PWRITE) + flags |= NVM_IO_SCRAMBLE_ENABLE; - ret = dev->ops->set_bb_tbl(dev, &rqd.ppa_addr, rqd.nr_ppas, type); - nvm_free_rqd_ppalist(tgt_dev, &rqd); - if (ret) { - pr_err("nvm: failed bb mark\n"); - return -EINVAL; - } - - return 0; + return flags; } -EXPORT_SYMBOL(nvm_set_tgt_bb_tbl); int nvm_submit_io(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd) { @@ -763,6 +740,7 @@ int nvm_submit_io(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd) nvm_rq_tgt_to_dev(tgt_dev, rqd); rqd->dev = tgt_dev; + rqd->flags = nvm_set_flags(&tgt_dev->geo, rqd); /* In case of error, fail with right address format */ ret = dev->ops->submit_io(dev, rqd); @@ -783,6 +761,7 @@ int nvm_submit_io_sync(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd) nvm_rq_tgt_to_dev(tgt_dev, rqd); rqd->dev = tgt_dev; + rqd->flags = nvm_set_flags(&tgt_dev->geo, rqd); /* In case of error, fail with right address format */ ret = dev->ops->submit_io_sync(dev, rqd); @@ -805,27 +784,159 @@ void nvm_end_io(struct nvm_rq *rqd) } EXPORT_SYMBOL(nvm_end_io); +static int nvm_submit_io_sync_raw(struct nvm_dev *dev, struct nvm_rq *rqd) +{ + if (!dev->ops->submit_io_sync) + return -ENODEV; + + rqd->flags = nvm_set_flags(&dev->geo, rqd); + + return dev->ops->submit_io_sync(dev, rqd); +} + +static int nvm_bb_chunk_sense(struct nvm_dev *dev, struct ppa_addr ppa) +{ + struct nvm_rq rqd = { NULL }; + struct bio bio; + struct bio_vec bio_vec; + struct page *page; + int ret; + + page = alloc_page(GFP_KERNEL); + if (!page) + return -ENOMEM; + + bio_init(&bio, &bio_vec, 1); + bio_add_page(&bio, page, PAGE_SIZE, 0); + bio_set_op_attrs(&bio, REQ_OP_READ, 0); + + rqd.bio = &bio; + rqd.opcode = NVM_OP_PREAD; + rqd.is_seq = 1; + rqd.nr_ppas = 1; + rqd.ppa_addr = generic_to_dev_addr(dev, ppa); + + ret = nvm_submit_io_sync_raw(dev, &rqd); + if (ret) + return ret; + + __free_page(page); + + return rqd.error; +} + /* - * folds a bad block list from its plane representation to its virtual - * block representation. The fold is done in place and reduced size is - * returned. - * - * If any of the planes status are bad or grown bad block, the virtual block - * is marked bad. If not bad, the first plane state acts as the block state. + * Scans a 1.2 chunk first and last page to determine if its state. + * If the chunk is found to be open, also scan it to update the write + * pointer. */ -int nvm_bb_tbl_fold(struct nvm_dev *dev, u8 *blks, int nr_blks) +static int nvm_bb_chunk_scan(struct nvm_dev *dev, struct ppa_addr ppa, + struct nvm_chk_meta *meta) { struct nvm_geo *geo = &dev->geo; - int blk, offset, pl, blktype; + int ret, pg, pl; - if (nr_blks != geo->num_chk * geo->pln_mode) - return -EINVAL; + /* sense first page */ + ret = nvm_bb_chunk_sense(dev, ppa); + if (ret < 0) /* io error */ + return ret; + else if (ret == 0) /* valid data */ + meta->state = NVM_CHK_ST_OPEN; + else if (ret > 0) { + /* + * If empty page, the chunk is free, else it is an + * actual io error. In that case, mark it offline. + */ + switch (ret) { + case NVM_RSP_ERR_EMPTYPAGE: + meta->state = NVM_CHK_ST_FREE; + return 0; + case NVM_RSP_ERR_FAILCRC: + case NVM_RSP_ERR_FAILECC: + case NVM_RSP_WARN_HIGHECC: + meta->state = NVM_CHK_ST_OPEN; + goto scan; + default: + return -ret; /* other io error */ + } + } + + /* sense last page */ + ppa.g.pg = geo->num_pg - 1; + ppa.g.pl = geo->num_pln - 1; + + ret = nvm_bb_chunk_sense(dev, ppa); + if (ret < 0) /* io error */ + return ret; + else if (ret == 0) { /* Chunk fully written */ + meta->state = NVM_CHK_ST_CLOSED; + meta->wp = geo->clba; + return 0; + } else if (ret > 0) { + switch (ret) { + case NVM_RSP_ERR_EMPTYPAGE: + case NVM_RSP_ERR_FAILCRC: + case NVM_RSP_ERR_FAILECC: + case NVM_RSP_WARN_HIGHECC: + meta->state = NVM_CHK_ST_OPEN; + break; + default: + return -ret; /* other io error */ + } + } + +scan: + /* + * chunk is open, we scan sequentially to update the write pointer. + * We make the assumption that targets write data across all planes + * before moving to the next page. + */ + for (pg = 0; pg < geo->num_pg; pg++) { + for (pl = 0; pl < geo->num_pln; pl++) { + ppa.g.pg = pg; + ppa.g.pl = pl; + + ret = nvm_bb_chunk_sense(dev, ppa); + if (ret < 0) /* io error */ + return ret; + else if (ret == 0) { + meta->wp += geo->ws_min; + } else if (ret > 0) { + switch (ret) { + case NVM_RSP_ERR_EMPTYPAGE: + return 0; + case NVM_RSP_ERR_FAILCRC: + case NVM_RSP_ERR_FAILECC: + case NVM_RSP_WARN_HIGHECC: + meta->wp += geo->ws_min; + break; + default: + return -ret; /* other io error */ + } + } + } + } + + return 0; +} + +/* + * folds a bad block list from its plane representation to its + * chunk representation. + * + * If any of the planes status are bad or grown bad, the chunk is marked + * offline. If not bad, the first plane state acts as the chunk state. + */ +static int nvm_bb_to_chunk(struct nvm_dev *dev, struct ppa_addr ppa, + u8 *blks, int nr_blks, struct nvm_chk_meta *meta) +{ + struct nvm_geo *geo = &dev->geo; + int ret, blk, pl, offset, blktype; for (blk = 0; blk < geo->num_chk; blk++) { offset = blk * geo->pln_mode; blktype = blks[offset]; - /* Bad blocks on any planes take precedence over other types */ for (pl = 0; pl < geo->pln_mode; pl++) { if (blks[offset + pl] & (NVM_BLK_T_BAD|NVM_BLK_T_GRWN_BAD)) { @@ -834,23 +945,124 @@ int nvm_bb_tbl_fold(struct nvm_dev *dev, u8 *blks, int nr_blks) } } - blks[blk] = blktype; + ppa.g.blk = blk; + + meta->wp = 0; + meta->type = NVM_CHK_TP_W_SEQ; + meta->wi = 0; + meta->slba = generic_to_dev_addr(dev, ppa).ppa; + meta->cnlb = dev->geo.clba; + + if (blktype == NVM_BLK_T_FREE) { + ret = nvm_bb_chunk_scan(dev, ppa, meta); + if (ret) + return ret; + } else { + meta->state = NVM_CHK_ST_OFFLINE; + } + + meta++; } - return geo->num_chk; + return 0; +} + +static int nvm_get_bb_meta(struct nvm_dev *dev, sector_t slba, + int nchks, struct nvm_chk_meta *meta) +{ + struct nvm_geo *geo = &dev->geo; + struct ppa_addr ppa; + u8 *blks; + int ch, lun, nr_blks; + int ret; + + ppa.ppa = slba; + ppa = dev_to_generic_addr(dev, ppa); + + if (ppa.g.blk != 0) + return -EINVAL; + + if ((nchks % geo->num_chk) != 0) + return -EINVAL; + + nr_blks = geo->num_chk * geo->pln_mode; + + blks = kmalloc(nr_blks, GFP_KERNEL); + if (!blks) + return -ENOMEM; + + for (ch = ppa.g.ch; ch < geo->num_ch; ch++) { + for (lun = ppa.g.lun; lun < geo->num_lun; lun++) { + struct ppa_addr ppa_gen, ppa_dev; + + if (!nchks) + goto done; + + ppa_gen.ppa = 0; + ppa_gen.g.ch = ch; + ppa_gen.g.lun = lun; + ppa_dev = generic_to_dev_addr(dev, ppa_gen); + + ret = dev->ops->get_bb_tbl(dev, ppa_dev, blks); + if (ret) + goto done; + + ret = nvm_bb_to_chunk(dev, ppa_gen, blks, nr_blks, + meta); + if (ret) + goto done; + + meta += geo->num_chk; + nchks -= geo->num_chk; + } + } +done: + kfree(blks); + return ret; } -EXPORT_SYMBOL(nvm_bb_tbl_fold); -int nvm_get_tgt_bb_tbl(struct nvm_tgt_dev *tgt_dev, struct ppa_addr ppa, - u8 *blks) +int nvm_get_chunk_meta(struct nvm_tgt_dev *tgt_dev, struct ppa_addr ppa, + int nchks, struct nvm_chk_meta *meta) { struct nvm_dev *dev = tgt_dev->parent; nvm_ppa_tgt_to_dev(tgt_dev, &ppa, 1); - return dev->ops->get_bb_tbl(dev, ppa, blks); + if (dev->geo.version == NVM_OCSSD_SPEC_12) + return nvm_get_bb_meta(dev, (sector_t)ppa.ppa, nchks, meta); + + return dev->ops->get_chk_meta(dev, (sector_t)ppa.ppa, nchks, meta); +} +EXPORT_SYMBOL_GPL(nvm_get_chunk_meta); + +int nvm_set_chunk_meta(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *ppas, + int nr_ppas, int type) +{ + struct nvm_dev *dev = tgt_dev->parent; + struct nvm_rq rqd; + int ret; + + if (dev->geo.version == NVM_OCSSD_SPEC_20) + return 0; + + if (nr_ppas > NVM_MAX_VLBA) { + pr_err("nvm: unable to update all blocks atomically\n"); + return -EINVAL; + } + + memset(&rqd, 0, sizeof(struct nvm_rq)); + + nvm_set_rqd_ppalist(tgt_dev, &rqd, ppas, nr_ppas); + nvm_rq_tgt_to_dev(tgt_dev, &rqd); + + ret = dev->ops->set_bb_tbl(dev, &rqd.ppa_addr, rqd.nr_ppas, type); + nvm_free_rqd_ppalist(tgt_dev, &rqd); + if (ret) + return -EINVAL; + + return 0; } -EXPORT_SYMBOL(nvm_get_tgt_bb_tbl); +EXPORT_SYMBOL_GPL(nvm_set_chunk_meta); static int nvm_core_init(struct nvm_dev *dev) { diff --git a/drivers/lightnvm/pblk-cache.c b/drivers/lightnvm/pblk-cache.c index f565a56b898a..c9fa26f95659 100644 --- a/drivers/lightnvm/pblk-cache.c +++ b/drivers/lightnvm/pblk-cache.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2016 CNEX Labs * Initial release: Javier Gonzalez <javier@cnexlabs.com> diff --git a/drivers/lightnvm/pblk-core.c b/drivers/lightnvm/pblk-core.c index 00984b486fea..6944aac43b01 100644 --- a/drivers/lightnvm/pblk-core.c +++ b/drivers/lightnvm/pblk-core.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2016 CNEX Labs * Initial release: Javier Gonzalez <javier@cnexlabs.com> @@ -16,7 +17,10 @@ * */ +#define CREATE_TRACE_POINTS + #include "pblk.h" +#include "pblk-trace.h" static void pblk_line_mark_bb(struct work_struct *work) { @@ -27,12 +31,12 @@ static void pblk_line_mark_bb(struct work_struct *work) struct ppa_addr *ppa = line_ws->priv; int ret; - ret = nvm_set_tgt_bb_tbl(dev, ppa, 1, NVM_BLK_T_GRWN_BAD); + ret = nvm_set_chunk_meta(dev, ppa, 1, NVM_BLK_T_GRWN_BAD); if (ret) { struct pblk_line *line; int pos; - line = &pblk->lines[pblk_ppa_to_line(*ppa)]; + line = pblk_ppa_to_line(pblk, *ppa); pos = pblk_ppa_to_pos(&dev->geo, *ppa); pblk_err(pblk, "failed to mark bb, line:%d, pos:%d\n", @@ -80,19 +84,28 @@ static void __pblk_end_io_erase(struct pblk *pblk, struct nvm_rq *rqd) struct pblk_line *line; int pos; - line = &pblk->lines[pblk_ppa_to_line(rqd->ppa_addr)]; + line = pblk_ppa_to_line(pblk, rqd->ppa_addr); pos = pblk_ppa_to_pos(geo, rqd->ppa_addr); chunk = &line->chks[pos]; atomic_dec(&line->left_seblks); if (rqd->error) { + trace_pblk_chunk_reset(pblk_disk_name(pblk), + &rqd->ppa_addr, PBLK_CHUNK_RESET_FAILED); + chunk->state = NVM_CHK_ST_OFFLINE; pblk_mark_bb(pblk, line, rqd->ppa_addr); } else { + trace_pblk_chunk_reset(pblk_disk_name(pblk), + &rqd->ppa_addr, PBLK_CHUNK_RESET_DONE); + chunk->state = NVM_CHK_ST_FREE; } + trace_pblk_chunk_state(pblk_disk_name(pblk), &rqd->ppa_addr, + chunk->state); + atomic_dec(&pblk->inflight_io); } @@ -108,9 +121,9 @@ static void pblk_end_io_erase(struct nvm_rq *rqd) /* * Get information for all chunks from the device. * - * The caller is responsible for freeing the returned structure + * The caller is responsible for freeing (vmalloc) the returned structure */ -struct nvm_chk_meta *pblk_chunk_get_info(struct pblk *pblk) +struct nvm_chk_meta *pblk_get_chunk_meta(struct pblk *pblk) { struct nvm_tgt_dev *dev = pblk->dev; struct nvm_geo *geo = &dev->geo; @@ -122,11 +135,11 @@ struct nvm_chk_meta *pblk_chunk_get_info(struct pblk *pblk) ppa.ppa = 0; len = geo->all_chunks * sizeof(*meta); - meta = kzalloc(len, GFP_KERNEL); + meta = vzalloc(len); if (!meta) return ERR_PTR(-ENOMEM); - ret = nvm_get_chunk_meta(dev, meta, ppa, geo->all_chunks); + ret = nvm_get_chunk_meta(dev, ppa, geo->all_chunks, meta); if (ret) { kfree(meta); return ERR_PTR(-EIO); @@ -192,7 +205,6 @@ void pblk_map_invalidate(struct pblk *pblk, struct ppa_addr ppa) { struct pblk_line *line; u64 paddr; - int line_id; #ifdef CONFIG_NVM_PBLK_DEBUG /* Callers must ensure that the ppa points to a device address */ @@ -200,8 +212,7 @@ void pblk_map_invalidate(struct pblk *pblk, struct ppa_addr ppa) BUG_ON(pblk_ppa_empty(ppa)); #endif - line_id = pblk_ppa_to_line(ppa); - line = &pblk->lines[line_id]; + line = pblk_ppa_to_line(pblk, ppa); paddr = pblk_dev_ppa_to_line_addr(pblk, ppa); __pblk_map_invalidate(pblk, line, paddr); @@ -227,6 +238,33 @@ static void pblk_invalidate_range(struct pblk *pblk, sector_t slba, spin_unlock(&pblk->trans_lock); } +int pblk_alloc_rqd_meta(struct pblk *pblk, struct nvm_rq *rqd) +{ + struct nvm_tgt_dev *dev = pblk->dev; + + rqd->meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, + &rqd->dma_meta_list); + if (!rqd->meta_list) + return -ENOMEM; + + if (rqd->nr_ppas == 1) + return 0; + + rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size; + rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size; + + return 0; +} + +void pblk_free_rqd_meta(struct pblk *pblk, struct nvm_rq *rqd) +{ + struct nvm_tgt_dev *dev = pblk->dev; + + if (rqd->meta_list) + nvm_dev_dma_free(dev->parent, rqd->meta_list, + rqd->dma_meta_list); +} + /* Caller must guarantee that the request is a valid type */ struct nvm_rq *pblk_alloc_rqd(struct pblk *pblk, int type) { @@ -258,7 +296,6 @@ struct nvm_rq *pblk_alloc_rqd(struct pblk *pblk, int type) /* Typically used on completion path. Cannot guarantee request consistency */ void pblk_free_rqd(struct pblk *pblk, struct nvm_rq *rqd, int type) { - struct nvm_tgt_dev *dev = pblk->dev; mempool_t *pool; switch (type) { @@ -279,9 +316,7 @@ void pblk_free_rqd(struct pblk *pblk, struct nvm_rq *rqd, int type) return; } - if (rqd->meta_list) - nvm_dev_dma_free(dev->parent, rqd->meta_list, - rqd->dma_meta_list); + pblk_free_rqd_meta(pblk, rqd); mempool_free(rqd, pool); } @@ -409,6 +444,9 @@ struct list_head *pblk_line_gc_list(struct pblk *pblk, struct pblk_line *line) } } else { line->state = PBLK_LINESTATE_CORRUPT; + trace_pblk_line_state(pblk_disk_name(pblk), line->id, + line->state); + line->gc_group = PBLK_LINEGC_NONE; move_list = &l_mg->corrupt_list; pblk_err(pblk, "corrupted vsc for line %d, vsc:%d (%d/%d/%d)\n", @@ -479,9 +517,30 @@ int pblk_submit_io(struct pblk *pblk, struct nvm_rq *rqd) return nvm_submit_io(dev, rqd); } +void pblk_check_chunk_state_update(struct pblk *pblk, struct nvm_rq *rqd) +{ + struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); + + int i; + + for (i = 0; i < rqd->nr_ppas; i++) { + struct ppa_addr *ppa = &ppa_list[i]; + struct nvm_chk_meta *chunk = pblk_dev_ppa_to_chunk(pblk, *ppa); + u64 caddr = pblk_dev_ppa_to_chunk_addr(pblk, *ppa); + + if (caddr == 0) + trace_pblk_chunk_state(pblk_disk_name(pblk), + ppa, NVM_CHK_ST_OPEN); + else if (caddr == chunk->cnlb) + trace_pblk_chunk_state(pblk_disk_name(pblk), + ppa, NVM_CHK_ST_CLOSED); + } +} + int pblk_submit_io_sync(struct pblk *pblk, struct nvm_rq *rqd) { struct nvm_tgt_dev *dev = pblk->dev; + int ret; atomic_inc(&pblk->inflight_io); @@ -490,7 +549,27 @@ int pblk_submit_io_sync(struct pblk *pblk, struct nvm_rq *rqd) return NVM_IO_ERR; #endif - return nvm_submit_io_sync(dev, rqd); + ret = nvm_submit_io_sync(dev, rqd); + + if (trace_pblk_chunk_state_enabled() && !ret && + rqd->opcode == NVM_OP_PWRITE) + pblk_check_chunk_state_update(pblk, rqd); + + return ret; +} + +int pblk_submit_io_sync_sem(struct pblk *pblk, struct nvm_rq *rqd) +{ + struct ppa_addr *ppa_list; + int ret; + + ppa_list = (rqd->nr_ppas > 1) ? rqd->ppa_list : &rqd->ppa_addr; + + pblk_down_chunk(pblk, ppa_list[0]); + ret = pblk_submit_io_sync(pblk, rqd); + pblk_up_chunk(pblk, ppa_list[0]); + + return ret; } static void pblk_bio_map_addr_endio(struct bio *bio) @@ -621,262 +700,227 @@ u64 pblk_lookup_page(struct pblk *pblk, struct pblk_line *line) return paddr; } -/* - * Submit emeta to one LUN in the raid line at the time to avoid a deadlock when - * taking the per LUN semaphore. - */ -static int pblk_line_submit_emeta_io(struct pblk *pblk, struct pblk_line *line, - void *emeta_buf, u64 paddr, int dir) +u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line) { struct nvm_tgt_dev *dev = pblk->dev; struct nvm_geo *geo = &dev->geo; - struct pblk_line_mgmt *l_mg = &pblk->l_mg; struct pblk_line_meta *lm = &pblk->lm; - void *ppa_list, *meta_list; - struct bio *bio; - struct nvm_rq rqd; - dma_addr_t dma_ppa_list, dma_meta_list; - int min = pblk->min_write_pgs; - int left_ppas = lm->emeta_sec[0]; - int id = line->id; - int rq_ppas, rq_len; - int cmd_op, bio_op; - int i, j; - int ret; + int bit; - if (dir == PBLK_WRITE) { - bio_op = REQ_OP_WRITE; - cmd_op = NVM_OP_PWRITE; - } else if (dir == PBLK_READ) { - bio_op = REQ_OP_READ; - cmd_op = NVM_OP_PREAD; - } else - return -EINVAL; + /* This usually only happens on bad lines */ + bit = find_first_zero_bit(line->blk_bitmap, lm->blk_per_line); + if (bit >= lm->blk_per_line) + return -1; - meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, - &dma_meta_list); - if (!meta_list) - return -ENOMEM; + return bit * geo->ws_opt; +} - ppa_list = meta_list + pblk_dma_meta_size; - dma_ppa_list = dma_meta_list + pblk_dma_meta_size; +int pblk_line_smeta_read(struct pblk *pblk, struct pblk_line *line) +{ + struct nvm_tgt_dev *dev = pblk->dev; + struct pblk_line_meta *lm = &pblk->lm; + struct bio *bio; + struct nvm_rq rqd; + u64 paddr = pblk_line_smeta_start(pblk, line); + int i, ret; -next_rq: memset(&rqd, 0, sizeof(struct nvm_rq)); - rq_ppas = pblk_calc_secs(pblk, left_ppas, 0); - rq_len = rq_ppas * geo->csecs; + ret = pblk_alloc_rqd_meta(pblk, &rqd); + if (ret) + return ret; - bio = pblk_bio_map_addr(pblk, emeta_buf, rq_ppas, rq_len, - l_mg->emeta_alloc_type, GFP_KERNEL); + bio = bio_map_kern(dev->q, line->smeta, lm->smeta_len, GFP_KERNEL); if (IS_ERR(bio)) { ret = PTR_ERR(bio); - goto free_rqd_dma; + goto clear_rqd; } bio->bi_iter.bi_sector = 0; /* internal bio */ - bio_set_op_attrs(bio, bio_op, 0); + bio_set_op_attrs(bio, REQ_OP_READ, 0); rqd.bio = bio; - rqd.meta_list = meta_list; - rqd.ppa_list = ppa_list; - rqd.dma_meta_list = dma_meta_list; - rqd.dma_ppa_list = dma_ppa_list; - rqd.opcode = cmd_op; - rqd.nr_ppas = rq_ppas; - - if (dir == PBLK_WRITE) { - struct pblk_sec_meta *meta_list = rqd.meta_list; - - rqd.flags = pblk_set_progr_mode(pblk, PBLK_WRITE); - for (i = 0; i < rqd.nr_ppas; ) { - spin_lock(&line->lock); - paddr = __pblk_alloc_page(pblk, line, min); - spin_unlock(&line->lock); - for (j = 0; j < min; j++, i++, paddr++) { - meta_list[i].lba = cpu_to_le64(ADDR_EMPTY); - rqd.ppa_list[i] = - addr_to_gen_ppa(pblk, paddr, id); - } - } - } else { - for (i = 0; i < rqd.nr_ppas; ) { - struct ppa_addr ppa = addr_to_gen_ppa(pblk, paddr, id); - int pos = pblk_ppa_to_pos(geo, ppa); - int read_type = PBLK_READ_RANDOM; - - if (pblk_io_aligned(pblk, rq_ppas)) - read_type = PBLK_READ_SEQUENTIAL; - rqd.flags = pblk_set_read_mode(pblk, read_type); - - while (test_bit(pos, line->blk_bitmap)) { - paddr += min; - if (pblk_boundary_paddr_checks(pblk, paddr)) { - pblk_err(pblk, "corrupt emeta line:%d\n", - line->id); - bio_put(bio); - ret = -EINTR; - goto free_rqd_dma; - } - - ppa = addr_to_gen_ppa(pblk, paddr, id); - pos = pblk_ppa_to_pos(geo, ppa); - } - - if (pblk_boundary_paddr_checks(pblk, paddr + min)) { - pblk_err(pblk, "corrupt emeta line:%d\n", - line->id); - bio_put(bio); - ret = -EINTR; - goto free_rqd_dma; - } + rqd.opcode = NVM_OP_PREAD; + rqd.nr_ppas = lm->smeta_sec; + rqd.is_seq = 1; - for (j = 0; j < min; j++, i++, paddr++) - rqd.ppa_list[i] = - addr_to_gen_ppa(pblk, paddr, line->id); - } - } + for (i = 0; i < lm->smeta_sec; i++, paddr++) + rqd.ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line->id); ret = pblk_submit_io_sync(pblk, &rqd); if (ret) { - pblk_err(pblk, "emeta I/O submission failed: %d\n", ret); + pblk_err(pblk, "smeta I/O submission failed: %d\n", ret); bio_put(bio); - goto free_rqd_dma; + goto clear_rqd; } atomic_dec(&pblk->inflight_io); - if (rqd.error) { - if (dir == PBLK_WRITE) - pblk_log_write_err(pblk, &rqd); - else - pblk_log_read_err(pblk, &rqd); - } + if (rqd.error) + pblk_log_read_err(pblk, &rqd); - emeta_buf += rq_len; - left_ppas -= rq_ppas; - if (left_ppas) - goto next_rq; -free_rqd_dma: - nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list); +clear_rqd: + pblk_free_rqd_meta(pblk, &rqd); return ret; } -u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line) -{ - struct nvm_tgt_dev *dev = pblk->dev; - struct nvm_geo *geo = &dev->geo; - struct pblk_line_meta *lm = &pblk->lm; - int bit; - - /* This usually only happens on bad lines */ - bit = find_first_zero_bit(line->blk_bitmap, lm->blk_per_line); - if (bit >= lm->blk_per_line) - return -1; - - return bit * geo->ws_opt; -} - -static int pblk_line_submit_smeta_io(struct pblk *pblk, struct pblk_line *line, - u64 paddr, int dir) +static int pblk_line_smeta_write(struct pblk *pblk, struct pblk_line *line, + u64 paddr) { struct nvm_tgt_dev *dev = pblk->dev; struct pblk_line_meta *lm = &pblk->lm; struct bio *bio; struct nvm_rq rqd; - __le64 *lba_list = NULL; + __le64 *lba_list = emeta_to_lbas(pblk, line->emeta->buf); + __le64 addr_empty = cpu_to_le64(ADDR_EMPTY); int i, ret; - int cmd_op, bio_op; - int flags; - - if (dir == PBLK_WRITE) { - bio_op = REQ_OP_WRITE; - cmd_op = NVM_OP_PWRITE; - flags = pblk_set_progr_mode(pblk, PBLK_WRITE); - lba_list = emeta_to_lbas(pblk, line->emeta->buf); - } else if (dir == PBLK_READ_RECOV || dir == PBLK_READ) { - bio_op = REQ_OP_READ; - cmd_op = NVM_OP_PREAD; - flags = pblk_set_read_mode(pblk, PBLK_READ_SEQUENTIAL); - } else - return -EINVAL; memset(&rqd, 0, sizeof(struct nvm_rq)); - rqd.meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, - &rqd.dma_meta_list); - if (!rqd.meta_list) - return -ENOMEM; - - rqd.ppa_list = rqd.meta_list + pblk_dma_meta_size; - rqd.dma_ppa_list = rqd.dma_meta_list + pblk_dma_meta_size; + ret = pblk_alloc_rqd_meta(pblk, &rqd); + if (ret) + return ret; bio = bio_map_kern(dev->q, line->smeta, lm->smeta_len, GFP_KERNEL); if (IS_ERR(bio)) { ret = PTR_ERR(bio); - goto free_ppa_list; + goto clear_rqd; } bio->bi_iter.bi_sector = 0; /* internal bio */ - bio_set_op_attrs(bio, bio_op, 0); + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); rqd.bio = bio; - rqd.opcode = cmd_op; - rqd.flags = flags; + rqd.opcode = NVM_OP_PWRITE; rqd.nr_ppas = lm->smeta_sec; + rqd.is_seq = 1; for (i = 0; i < lm->smeta_sec; i++, paddr++) { struct pblk_sec_meta *meta_list = rqd.meta_list; rqd.ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line->id); - - if (dir == PBLK_WRITE) { - __le64 addr_empty = cpu_to_le64(ADDR_EMPTY); - - meta_list[i].lba = lba_list[paddr] = addr_empty; - } + meta_list[i].lba = lba_list[paddr] = addr_empty; } - /* - * This I/O is sent by the write thread when a line is replace. Since - * the write thread is the only one sending write and erase commands, - * there is no need to take the LUN semaphore. - */ - ret = pblk_submit_io_sync(pblk, &rqd); + ret = pblk_submit_io_sync_sem(pblk, &rqd); if (ret) { pblk_err(pblk, "smeta I/O submission failed: %d\n", ret); bio_put(bio); - goto free_ppa_list; + goto clear_rqd; } atomic_dec(&pblk->inflight_io); if (rqd.error) { - if (dir == PBLK_WRITE) { - pblk_log_write_err(pblk, &rqd); - ret = 1; - } else if (dir == PBLK_READ) - pblk_log_read_err(pblk, &rqd); + pblk_log_write_err(pblk, &rqd); + ret = -EIO; } -free_ppa_list: - nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list); - +clear_rqd: + pblk_free_rqd_meta(pblk, &rqd); return ret; } -int pblk_line_read_smeta(struct pblk *pblk, struct pblk_line *line) +int pblk_line_emeta_read(struct pblk *pblk, struct pblk_line *line, + void *emeta_buf) { - u64 bpaddr = pblk_line_smeta_start(pblk, line); + struct nvm_tgt_dev *dev = pblk->dev; + struct nvm_geo *geo = &dev->geo; + struct pblk_line_mgmt *l_mg = &pblk->l_mg; + struct pblk_line_meta *lm = &pblk->lm; + void *ppa_list, *meta_list; + struct bio *bio; + struct nvm_rq rqd; + u64 paddr = line->emeta_ssec; + dma_addr_t dma_ppa_list, dma_meta_list; + int min = pblk->min_write_pgs; + int left_ppas = lm->emeta_sec[0]; + int line_id = line->id; + int rq_ppas, rq_len; + int i, j; + int ret; - return pblk_line_submit_smeta_io(pblk, line, bpaddr, PBLK_READ_RECOV); -} + meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, + &dma_meta_list); + if (!meta_list) + return -ENOMEM; -int pblk_line_read_emeta(struct pblk *pblk, struct pblk_line *line, - void *emeta_buf) -{ - return pblk_line_submit_emeta_io(pblk, line, emeta_buf, - line->emeta_ssec, PBLK_READ); + ppa_list = meta_list + pblk_dma_meta_size; + dma_ppa_list = dma_meta_list + pblk_dma_meta_size; + +next_rq: + memset(&rqd, 0, sizeof(struct nvm_rq)); + + rq_ppas = pblk_calc_secs(pblk, left_ppas, 0); + rq_len = rq_ppas * geo->csecs; + + bio = pblk_bio_map_addr(pblk, emeta_buf, rq_ppas, rq_len, + l_mg->emeta_alloc_type, GFP_KERNEL); + if (IS_ERR(bio)) { + ret = PTR_ERR(bio); + goto free_rqd_dma; + } + + bio->bi_iter.bi_sector = 0; /* internal bio */ + bio_set_op_attrs(bio, REQ_OP_READ, 0); + + rqd.bio = bio; + rqd.meta_list = meta_list; + rqd.ppa_list = ppa_list; + rqd.dma_meta_list = dma_meta_list; + rqd.dma_ppa_list = dma_ppa_list; + rqd.opcode = NVM_OP_PREAD; + rqd.nr_ppas = rq_ppas; + + for (i = 0; i < rqd.nr_ppas; ) { + struct ppa_addr ppa = addr_to_gen_ppa(pblk, paddr, line_id); + int pos = pblk_ppa_to_pos(geo, ppa); + + if (pblk_io_aligned(pblk, rq_ppas)) + rqd.is_seq = 1; + + while (test_bit(pos, line->blk_bitmap)) { + paddr += min; + if (pblk_boundary_paddr_checks(pblk, paddr)) { + bio_put(bio); + ret = -EINTR; + goto free_rqd_dma; + } + + ppa = addr_to_gen_ppa(pblk, paddr, line_id); + pos = pblk_ppa_to_pos(geo, ppa); + } + + if (pblk_boundary_paddr_checks(pblk, paddr + min)) { + bio_put(bio); + ret = -EINTR; + goto free_rqd_dma; + } + + for (j = 0; j < min; j++, i++, paddr++) + rqd.ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line_id); + } + + ret = pblk_submit_io_sync(pblk, &rqd); + if (ret) { + pblk_err(pblk, "emeta I/O submission failed: %d\n", ret); + bio_put(bio); + goto free_rqd_dma; + } + + atomic_dec(&pblk->inflight_io); + + if (rqd.error) + pblk_log_read_err(pblk, &rqd); + + emeta_buf += rq_len; + left_ppas -= rq_ppas; + if (left_ppas) + goto next_rq; + +free_rqd_dma: + nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list); + return ret; } static void pblk_setup_e_rq(struct pblk *pblk, struct nvm_rq *rqd, @@ -885,16 +929,17 @@ static void pblk_setup_e_rq(struct pblk *pblk, struct nvm_rq *rqd, rqd->opcode = NVM_OP_ERASE; rqd->ppa_addr = ppa; rqd->nr_ppas = 1; - rqd->flags = pblk_set_progr_mode(pblk, PBLK_ERASE); + rqd->is_seq = 1; rqd->bio = NULL; } static int pblk_blk_erase_sync(struct pblk *pblk, struct ppa_addr ppa) { - struct nvm_rq rqd; - int ret = 0; + struct nvm_rq rqd = {NULL}; + int ret; - memset(&rqd, 0, sizeof(struct nvm_rq)); + trace_pblk_chunk_reset(pblk_disk_name(pblk), &ppa, + PBLK_CHUNK_RESET_START); pblk_setup_e_rq(pblk, &rqd, ppa); @@ -902,19 +947,6 @@ static int pblk_blk_erase_sync(struct pblk *pblk, struct ppa_addr ppa) * with writes. Thus, there is no need to take the LUN semaphore. */ ret = pblk_submit_io_sync(pblk, &rqd); - if (ret) { - struct nvm_tgt_dev *dev = pblk->dev; - struct nvm_geo *geo = &dev->geo; - - pblk_err(pblk, "could not sync erase line:%d,blk:%d\n", - pblk_ppa_to_line(ppa), - pblk_ppa_to_pos(geo, ppa)); - - rqd.error = ret; - goto out; - } - -out: rqd.private = pblk; __pblk_end_io_erase(pblk, &rqd); @@ -1008,6 +1040,8 @@ static int pblk_line_init_metadata(struct pblk *pblk, struct pblk_line *line, spin_lock(&l_mg->free_lock); spin_lock(&line->lock); line->state = PBLK_LINESTATE_BAD; + trace_pblk_line_state(pblk_disk_name(pblk), line->id, + line->state); spin_unlock(&line->lock); list_add_tail(&line->list, &l_mg->bad_list); @@ -1071,15 +1105,18 @@ static int pblk_line_init_metadata(struct pblk *pblk, struct pblk_line *line, static int pblk_line_alloc_bitmaps(struct pblk *pblk, struct pblk_line *line) { struct pblk_line_meta *lm = &pblk->lm; + struct pblk_line_mgmt *l_mg = &pblk->l_mg; - line->map_bitmap = kzalloc(lm->sec_bitmap_len, GFP_KERNEL); + line->map_bitmap = mempool_alloc(l_mg->bitmap_pool, GFP_KERNEL); if (!line->map_bitmap) return -ENOMEM; + memset(line->map_bitmap, 0, lm->sec_bitmap_len); + /* will be initialized using bb info from map_bitmap */ - line->invalid_bitmap = kmalloc(lm->sec_bitmap_len, GFP_KERNEL); + line->invalid_bitmap = mempool_alloc(l_mg->bitmap_pool, GFP_KERNEL); if (!line->invalid_bitmap) { - kfree(line->map_bitmap); + mempool_free(line->map_bitmap, l_mg->bitmap_pool); line->map_bitmap = NULL; return -ENOMEM; } @@ -1122,7 +1159,7 @@ static int pblk_line_init_bb(struct pblk *pblk, struct pblk_line *line, line->smeta_ssec = off; line->cur_sec = off + lm->smeta_sec; - if (init && pblk_line_submit_smeta_io(pblk, line, off, PBLK_WRITE)) { + if (init && pblk_line_smeta_write(pblk, line, off)) { pblk_debug(pblk, "line smeta I/O failed. Retry\n"); return 0; } @@ -1152,6 +1189,8 @@ static int pblk_line_init_bb(struct pblk *pblk, struct pblk_line *line, bitmap_weight(line->invalid_bitmap, lm->sec_per_line)) { spin_lock(&line->lock); line->state = PBLK_LINESTATE_BAD; + trace_pblk_line_state(pblk_disk_name(pblk), line->id, + line->state); spin_unlock(&line->lock); list_add_tail(&line->list, &l_mg->bad_list); @@ -1204,6 +1243,8 @@ static int pblk_line_prepare(struct pblk *pblk, struct pblk_line *line) if (line->state == PBLK_LINESTATE_NEW) { blk_to_erase = pblk_prepare_new_line(pblk, line); line->state = PBLK_LINESTATE_FREE; + trace_pblk_line_state(pblk_disk_name(pblk), line->id, + line->state); } else { blk_to_erase = blk_in_line; } @@ -1221,6 +1262,8 @@ static int pblk_line_prepare(struct pblk *pblk, struct pblk_line *line) } line->state = PBLK_LINESTATE_OPEN; + trace_pblk_line_state(pblk_disk_name(pblk), line->id, + line->state); atomic_set(&line->left_eblks, blk_to_erase); atomic_set(&line->left_seblks, blk_to_erase); @@ -1265,7 +1308,9 @@ int pblk_line_recov_alloc(struct pblk *pblk, struct pblk_line *line) void pblk_line_recov_close(struct pblk *pblk, struct pblk_line *line) { - kfree(line->map_bitmap); + struct pblk_line_mgmt *l_mg = &pblk->l_mg; + + mempool_free(line->map_bitmap, l_mg->bitmap_pool); line->map_bitmap = NULL; line->smeta = NULL; line->emeta = NULL; @@ -1283,8 +1328,11 @@ static void pblk_line_reinit(struct pblk_line *line) void pblk_line_free(struct pblk_line *line) { - kfree(line->map_bitmap); - kfree(line->invalid_bitmap); + struct pblk *pblk = line->pblk; + struct pblk_line_mgmt *l_mg = &pblk->l_mg; + + mempool_free(line->map_bitmap, l_mg->bitmap_pool); + mempool_free(line->invalid_bitmap, l_mg->bitmap_pool); pblk_line_reinit(line); } @@ -1312,6 +1360,8 @@ retry: if (unlikely(bit >= lm->blk_per_line)) { spin_lock(&line->lock); line->state = PBLK_LINESTATE_BAD; + trace_pblk_line_state(pblk_disk_name(pblk), line->id, + line->state); spin_unlock(&line->lock); list_add_tail(&line->list, &l_mg->bad_list); @@ -1446,12 +1496,32 @@ retry_setup: return line; } +void pblk_ppa_to_line_put(struct pblk *pblk, struct ppa_addr ppa) +{ + struct pblk_line *line; + + line = pblk_ppa_to_line(pblk, ppa); + kref_put(&line->ref, pblk_line_put_wq); +} + +void pblk_rq_to_line_put(struct pblk *pblk, struct nvm_rq *rqd) +{ + struct ppa_addr *ppa_list; + int i; + + ppa_list = (rqd->nr_ppas > 1) ? rqd->ppa_list : &rqd->ppa_addr; + + for (i = 0; i < rqd->nr_ppas; i++) + pblk_ppa_to_line_put(pblk, ppa_list[i]); +} + static void pblk_stop_writes(struct pblk *pblk, struct pblk_line *line) { lockdep_assert_held(&pblk->l_mg.free_lock); pblk_set_space_limit(pblk); pblk->state = PBLK_STATE_STOPPING; + trace_pblk_state(pblk_disk_name(pblk), pblk->state); } static void pblk_line_close_meta_sync(struct pblk *pblk) @@ -1501,6 +1571,7 @@ void __pblk_pipeline_flush(struct pblk *pblk) return; } pblk->state = PBLK_STATE_RECOVERING; + trace_pblk_state(pblk_disk_name(pblk), pblk->state); spin_unlock(&l_mg->free_lock); pblk_flush_writer(pblk); @@ -1522,6 +1593,7 @@ void __pblk_pipeline_stop(struct pblk *pblk) spin_lock(&l_mg->free_lock); pblk->state = PBLK_STATE_STOPPED; + trace_pblk_state(pblk_disk_name(pblk), pblk->state); l_mg->data_line = NULL; l_mg->data_next = NULL; spin_unlock(&l_mg->free_lock); @@ -1539,13 +1611,14 @@ struct pblk_line *pblk_line_replace_data(struct pblk *pblk) struct pblk_line *cur, *new = NULL; unsigned int left_seblks; - cur = l_mg->data_line; new = l_mg->data_next; if (!new) goto out; - l_mg->data_line = new; spin_lock(&l_mg->free_lock); + cur = l_mg->data_line; + l_mg->data_line = new; + pblk_line_setup_metadata(new, l_mg, &pblk->lm); spin_unlock(&l_mg->free_lock); @@ -1612,6 +1685,8 @@ static void __pblk_line_put(struct pblk *pblk, struct pblk_line *line) spin_lock(&line->lock); WARN_ON(line->state != PBLK_LINESTATE_GC); line->state = PBLK_LINESTATE_FREE; + trace_pblk_line_state(pblk_disk_name(pblk), line->id, + line->state); line->gc_group = PBLK_LINEGC_NONE; pblk_line_free(line); @@ -1680,6 +1755,9 @@ int pblk_blk_erase_async(struct pblk *pblk, struct ppa_addr ppa) rqd->end_io = pblk_end_io_erase; rqd->private = pblk; + trace_pblk_chunk_reset(pblk_disk_name(pblk), + &ppa, PBLK_CHUNK_RESET_START); + /* The write thread schedules erases so that it minimizes disturbances * with writes. Thus, there is no need to take the LUN semaphore. */ @@ -1689,7 +1767,7 @@ int pblk_blk_erase_async(struct pblk *pblk, struct ppa_addr ppa) struct nvm_geo *geo = &dev->geo; pblk_err(pblk, "could not async erase line:%d,blk:%d\n", - pblk_ppa_to_line(ppa), + pblk_ppa_to_line_id(ppa), pblk_ppa_to_pos(geo, ppa)); } @@ -1741,10 +1819,9 @@ void pblk_line_close(struct pblk *pblk, struct pblk_line *line) WARN_ON(line->state != PBLK_LINESTATE_OPEN); line->state = PBLK_LINESTATE_CLOSED; move_list = pblk_line_gc_list(pblk, line); - list_add_tail(&line->list, move_list); - kfree(line->map_bitmap); + mempool_free(line->map_bitmap, l_mg->bitmap_pool); line->map_bitmap = NULL; line->smeta = NULL; line->emeta = NULL; @@ -1760,6 +1837,9 @@ void pblk_line_close(struct pblk *pblk, struct pblk_line *line) spin_unlock(&line->lock); spin_unlock(&l_mg->gc_lock); + + trace_pblk_line_state(pblk_disk_name(pblk), line->id, + line->state); } void pblk_line_close_meta(struct pblk *pblk, struct pblk_line *line) @@ -1778,6 +1858,17 @@ void pblk_line_close_meta(struct pblk *pblk, struct pblk_line *line) wa->pad = cpu_to_le64(atomic64_read(&pblk->pad_wa)); wa->gc = cpu_to_le64(atomic64_read(&pblk->gc_wa)); + if (le32_to_cpu(emeta_buf->header.identifier) != PBLK_MAGIC) { + emeta_buf->header.identifier = cpu_to_le32(PBLK_MAGIC); + memcpy(emeta_buf->header.uuid, pblk->instance_uuid, 16); + emeta_buf->header.id = cpu_to_le32(line->id); + emeta_buf->header.type = cpu_to_le16(line->type); + emeta_buf->header.version_major = EMETA_VERSION_MAJOR; + emeta_buf->header.version_minor = EMETA_VERSION_MINOR; + emeta_buf->header.crc = cpu_to_le32( + pblk_calc_meta_header_crc(pblk, &emeta_buf->header)); + } + emeta_buf->nr_valid_lbas = cpu_to_le64(line->nr_valid_lbas); emeta_buf->crc = cpu_to_le32(pblk_calc_emeta_crc(pblk, emeta_buf)); @@ -1795,8 +1886,6 @@ void pblk_line_close_meta(struct pblk *pblk, struct pblk_line *line) spin_unlock(&l_mg->close_lock); pblk_line_should_sync_meta(pblk); - - } static void pblk_save_lba_list(struct pblk *pblk, struct pblk_line *line) @@ -1847,8 +1936,7 @@ void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv, queue_work(wq, &line_ws->ws); } -static void __pblk_down_page(struct pblk *pblk, struct ppa_addr *ppa_list, - int nr_ppas, int pos) +static void __pblk_down_chunk(struct pblk *pblk, int pos) { struct pblk_lun *rlun = &pblk->luns[pos]; int ret; @@ -1857,13 +1945,6 @@ static void __pblk_down_page(struct pblk *pblk, struct ppa_addr *ppa_list, * Only send one inflight I/O per LUN. Since we map at a page * granurality, all ppas in the I/O will map to the same LUN */ -#ifdef CONFIG_NVM_PBLK_DEBUG - int i; - - for (i = 1; i < nr_ppas; i++) - WARN_ON(ppa_list[0].a.lun != ppa_list[i].a.lun || - ppa_list[0].a.ch != ppa_list[i].a.ch); -#endif ret = down_timeout(&rlun->wr_sem, msecs_to_jiffies(30000)); if (ret == -ETIME || ret == -EINTR) @@ -1871,21 +1952,21 @@ static void __pblk_down_page(struct pblk *pblk, struct ppa_addr *ppa_list, -ret); } -void pblk_down_page(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas) +void pblk_down_chunk(struct pblk *pblk, struct ppa_addr ppa) { struct nvm_tgt_dev *dev = pblk->dev; struct nvm_geo *geo = &dev->geo; - int pos = pblk_ppa_to_pos(geo, ppa_list[0]); + int pos = pblk_ppa_to_pos(geo, ppa); - __pblk_down_page(pblk, ppa_list, nr_ppas, pos); + __pblk_down_chunk(pblk, pos); } -void pblk_down_rq(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas, +void pblk_down_rq(struct pblk *pblk, struct ppa_addr ppa, unsigned long *lun_bitmap) { struct nvm_tgt_dev *dev = pblk->dev; struct nvm_geo *geo = &dev->geo; - int pos = pblk_ppa_to_pos(geo, ppa_list[0]); + int pos = pblk_ppa_to_pos(geo, ppa); /* If the LUN has been locked for this same request, do no attempt to * lock it again @@ -1893,30 +1974,21 @@ void pblk_down_rq(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas, if (test_and_set_bit(pos, lun_bitmap)) return; - __pblk_down_page(pblk, ppa_list, nr_ppas, pos); + __pblk_down_chunk(pblk, pos); } -void pblk_up_page(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas) +void pblk_up_chunk(struct pblk *pblk, struct ppa_addr ppa) { struct nvm_tgt_dev *dev = pblk->dev; struct nvm_geo *geo = &dev->geo; struct pblk_lun *rlun; - int pos = pblk_ppa_to_pos(geo, ppa_list[0]); - -#ifdef CONFIG_NVM_PBLK_DEBUG - int i; - - for (i = 1; i < nr_ppas; i++) - WARN_ON(ppa_list[0].a.lun != ppa_list[i].a.lun || - ppa_list[0].a.ch != ppa_list[i].a.ch); -#endif + int pos = pblk_ppa_to_pos(geo, ppa); rlun = &pblk->luns[pos]; up(&rlun->wr_sem); } -void pblk_up_rq(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas, - unsigned long *lun_bitmap) +void pblk_up_rq(struct pblk *pblk, unsigned long *lun_bitmap) { struct nvm_tgt_dev *dev = pblk->dev; struct nvm_geo *geo = &dev->geo; @@ -2060,8 +2132,7 @@ void pblk_lookup_l2p_seq(struct pblk *pblk, struct ppa_addr *ppas, /* If the L2P entry maps to a line, the reference is valid */ if (!pblk_ppa_empty(ppa) && !pblk_addr_in_cache(ppa)) { - int line_id = pblk_ppa_to_line(ppa); - struct pblk_line *line = &pblk->lines[line_id]; + struct pblk_line *line = pblk_ppa_to_line(pblk, ppa); kref_get(&line->ref); } diff --git a/drivers/lightnvm/pblk-gc.c b/drivers/lightnvm/pblk-gc.c index 157c2567c9e8..2fa118c8eb71 100644 --- a/drivers/lightnvm/pblk-gc.c +++ b/drivers/lightnvm/pblk-gc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2016 CNEX Labs * Initial release: Javier Gonzalez <javier@cnexlabs.com> @@ -16,8 +17,10 @@ */ #include "pblk.h" +#include "pblk-trace.h" #include <linux/delay.h> + static void pblk_gc_free_gc_rq(struct pblk_gc_rq *gc_rq) { if (gc_rq->data) @@ -64,6 +67,8 @@ static void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line) spin_lock(&line->lock); WARN_ON(line->state != PBLK_LINESTATE_GC); line->state = PBLK_LINESTATE_CLOSED; + trace_pblk_line_state(pblk_disk_name(pblk), line->id, + line->state); move_list = pblk_line_gc_list(pblk, line); spin_unlock(&line->lock); @@ -144,7 +149,7 @@ static __le64 *get_lba_list_from_emeta(struct pblk *pblk, if (!emeta_buf) return NULL; - ret = pblk_line_read_emeta(pblk, line, emeta_buf); + ret = pblk_line_emeta_read(pblk, line, emeta_buf); if (ret) { pblk_err(pblk, "line %d read emeta failed (%d)\n", line->id, ret); @@ -405,6 +410,8 @@ void pblk_gc_free_full_lines(struct pblk *pblk) spin_lock(&line->lock); WARN_ON(line->state != PBLK_LINESTATE_CLOSED); line->state = PBLK_LINESTATE_GC; + trace_pblk_line_state(pblk_disk_name(pblk), line->id, + line->state); spin_unlock(&line->lock); list_del(&line->list); @@ -451,6 +458,8 @@ next_gc_group: spin_lock(&line->lock); WARN_ON(line->state != PBLK_LINESTATE_CLOSED); line->state = PBLK_LINESTATE_GC; + trace_pblk_line_state(pblk_disk_name(pblk), line->id, + line->state); spin_unlock(&line->lock); list_del(&line->list); diff --git a/drivers/lightnvm/pblk-init.c b/drivers/lightnvm/pblk-init.c index 537e98f2b24a..13822594647c 100644 --- a/drivers/lightnvm/pblk-init.c +++ b/drivers/lightnvm/pblk-init.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2015 IT University of Copenhagen (rrpc.c) * Copyright (C) 2016 CNEX Labs @@ -19,15 +20,31 @@ */ #include "pblk.h" +#include "pblk-trace.h" static unsigned int write_buffer_size; module_param(write_buffer_size, uint, 0644); MODULE_PARM_DESC(write_buffer_size, "number of entries in a write buffer"); -static struct kmem_cache *pblk_ws_cache, *pblk_rec_cache, *pblk_g_rq_cache, - *pblk_w_rq_cache; -static DECLARE_RWSEM(pblk_lock); +struct pblk_global_caches { + struct kmem_cache *ws; + struct kmem_cache *rec; + struct kmem_cache *g_rq; + struct kmem_cache *w_rq; + + struct kref kref; + + struct mutex mutex; /* Ensures consistency between + * caches and kref + */ +}; + +static struct pblk_global_caches pblk_caches = { + .mutex = __MUTEX_INITIALIZER(pblk_caches.mutex), + .kref = KREF_INIT(0), +}; + struct bio_set pblk_bio_set; static int pblk_rw_io(struct request_queue *q, struct pblk *pblk, @@ -168,36 +185,26 @@ static void pblk_rwb_free(struct pblk *pblk) if (pblk_rb_tear_down_check(&pblk->rwb)) pblk_err(pblk, "write buffer error on tear down\n"); - pblk_rb_data_free(&pblk->rwb); - vfree(pblk_rb_entries_ref(&pblk->rwb)); + pblk_rb_free(&pblk->rwb); } static int pblk_rwb_init(struct pblk *pblk) { struct nvm_tgt_dev *dev = pblk->dev; struct nvm_geo *geo = &dev->geo; - struct pblk_rb_entry *entries; - unsigned long nr_entries, buffer_size; - unsigned int power_size, power_seg_sz; - int pgs_in_buffer; + unsigned long buffer_size; + int pgs_in_buffer, threshold; - pgs_in_buffer = max(geo->mw_cunits, geo->ws_opt) * geo->all_luns; + threshold = geo->mw_cunits * geo->all_luns; + pgs_in_buffer = (max(geo->mw_cunits, geo->ws_opt) + geo->ws_opt) + * geo->all_luns; if (write_buffer_size && (write_buffer_size > pgs_in_buffer)) buffer_size = write_buffer_size; else buffer_size = pgs_in_buffer; - nr_entries = pblk_rb_calculate_size(buffer_size); - - entries = vzalloc(array_size(nr_entries, sizeof(struct pblk_rb_entry))); - if (!entries) - return -ENOMEM; - - power_size = get_count_order(nr_entries); - power_seg_sz = get_count_order(geo->csecs); - - return pblk_rb_init(&pblk->rwb, entries, power_size, power_seg_sz); + return pblk_rb_init(&pblk->rwb, buffer_size, threshold, geo->csecs); } /* Minimum pages needed within a lun */ @@ -306,53 +313,80 @@ static int pblk_set_addrf(struct pblk *pblk) return 0; } -static int pblk_init_global_caches(struct pblk *pblk) +static int pblk_create_global_caches(void) { - down_write(&pblk_lock); - pblk_ws_cache = kmem_cache_create("pblk_blk_ws", + + pblk_caches.ws = kmem_cache_create("pblk_blk_ws", sizeof(struct pblk_line_ws), 0, 0, NULL); - if (!pblk_ws_cache) { - up_write(&pblk_lock); + if (!pblk_caches.ws) return -ENOMEM; - } - pblk_rec_cache = kmem_cache_create("pblk_rec", + pblk_caches.rec = kmem_cache_create("pblk_rec", sizeof(struct pblk_rec_ctx), 0, 0, NULL); - if (!pblk_rec_cache) { - kmem_cache_destroy(pblk_ws_cache); - up_write(&pblk_lock); - return -ENOMEM; - } + if (!pblk_caches.rec) + goto fail_destroy_ws; - pblk_g_rq_cache = kmem_cache_create("pblk_g_rq", pblk_g_rq_size, + pblk_caches.g_rq = kmem_cache_create("pblk_g_rq", pblk_g_rq_size, 0, 0, NULL); - if (!pblk_g_rq_cache) { - kmem_cache_destroy(pblk_ws_cache); - kmem_cache_destroy(pblk_rec_cache); - up_write(&pblk_lock); - return -ENOMEM; - } + if (!pblk_caches.g_rq) + goto fail_destroy_rec; - pblk_w_rq_cache = kmem_cache_create("pblk_w_rq", pblk_w_rq_size, + pblk_caches.w_rq = kmem_cache_create("pblk_w_rq", pblk_w_rq_size, 0, 0, NULL); - if (!pblk_w_rq_cache) { - kmem_cache_destroy(pblk_ws_cache); - kmem_cache_destroy(pblk_rec_cache); - kmem_cache_destroy(pblk_g_rq_cache); - up_write(&pblk_lock); - return -ENOMEM; - } - up_write(&pblk_lock); + if (!pblk_caches.w_rq) + goto fail_destroy_g_rq; return 0; + +fail_destroy_g_rq: + kmem_cache_destroy(pblk_caches.g_rq); +fail_destroy_rec: + kmem_cache_destroy(pblk_caches.rec); +fail_destroy_ws: + kmem_cache_destroy(pblk_caches.ws); + + return -ENOMEM; } -static void pblk_free_global_caches(struct pblk *pblk) +static int pblk_get_global_caches(void) { - kmem_cache_destroy(pblk_ws_cache); - kmem_cache_destroy(pblk_rec_cache); - kmem_cache_destroy(pblk_g_rq_cache); - kmem_cache_destroy(pblk_w_rq_cache); + int ret; + + mutex_lock(&pblk_caches.mutex); + + if (kref_read(&pblk_caches.kref) > 0) { + kref_get(&pblk_caches.kref); + mutex_unlock(&pblk_caches.mutex); + return 0; + } + + ret = pblk_create_global_caches(); + + if (!ret) + kref_get(&pblk_caches.kref); + + mutex_unlock(&pblk_caches.mutex); + + return ret; +} + +static void pblk_destroy_global_caches(struct kref *ref) +{ + struct pblk_global_caches *c; + + c = container_of(ref, struct pblk_global_caches, kref); + + kmem_cache_destroy(c->ws); + kmem_cache_destroy(c->rec); + kmem_cache_destroy(c->g_rq); + kmem_cache_destroy(c->w_rq); +} + +static void pblk_put_global_caches(void) +{ + mutex_lock(&pblk_caches.mutex); + kref_put(&pblk_caches.kref, pblk_destroy_global_caches); + mutex_unlock(&pblk_caches.mutex); } static int pblk_core_init(struct pblk *pblk) @@ -371,23 +405,19 @@ static int pblk_core_init(struct pblk *pblk) atomic64_set(&pblk->nr_flush, 0); pblk->nr_flush_rst = 0; - pblk->min_write_pgs = geo->ws_opt * (geo->csecs / PAGE_SIZE); + pblk->min_write_pgs = geo->ws_opt; max_write_ppas = pblk->min_write_pgs * geo->all_luns; pblk->max_write_pgs = min_t(int, max_write_ppas, NVM_MAX_VLBA); + pblk->max_write_pgs = min_t(int, pblk->max_write_pgs, + queue_max_hw_sectors(dev->q) / (geo->csecs >> SECTOR_SHIFT)); pblk_set_sec_per_write(pblk, pblk->min_write_pgs); - if (pblk->max_write_pgs > PBLK_MAX_REQ_ADDRS) { - pblk_err(pblk, "vector list too big(%u > %u)\n", - pblk->max_write_pgs, PBLK_MAX_REQ_ADDRS); - return -EINVAL; - } - pblk->pad_dist = kcalloc(pblk->min_write_pgs - 1, sizeof(atomic64_t), GFP_KERNEL); if (!pblk->pad_dist) return -ENOMEM; - if (pblk_init_global_caches(pblk)) + if (pblk_get_global_caches()) goto fail_free_pad_dist; /* Internal bios can be at most the sectors signaled by the device. */ @@ -396,27 +426,27 @@ static int pblk_core_init(struct pblk *pblk) goto free_global_caches; ret = mempool_init_slab_pool(&pblk->gen_ws_pool, PBLK_GEN_WS_POOL_SIZE, - pblk_ws_cache); + pblk_caches.ws); if (ret) goto free_page_bio_pool; ret = mempool_init_slab_pool(&pblk->rec_pool, geo->all_luns, - pblk_rec_cache); + pblk_caches.rec); if (ret) goto free_gen_ws_pool; ret = mempool_init_slab_pool(&pblk->r_rq_pool, geo->all_luns, - pblk_g_rq_cache); + pblk_caches.g_rq); if (ret) goto free_rec_pool; ret = mempool_init_slab_pool(&pblk->e_rq_pool, geo->all_luns, - pblk_g_rq_cache); + pblk_caches.g_rq); if (ret) goto free_r_rq_pool; ret = mempool_init_slab_pool(&pblk->w_rq_pool, geo->all_luns, - pblk_w_rq_cache); + pblk_caches.w_rq); if (ret) goto free_e_rq_pool; @@ -462,7 +492,7 @@ free_gen_ws_pool: free_page_bio_pool: mempool_exit(&pblk->page_bio_pool); free_global_caches: - pblk_free_global_caches(pblk); + pblk_put_global_caches(); fail_free_pad_dist: kfree(pblk->pad_dist); return -ENOMEM; @@ -486,7 +516,7 @@ static void pblk_core_free(struct pblk *pblk) mempool_exit(&pblk->e_rq_pool); mempool_exit(&pblk->w_rq_pool); - pblk_free_global_caches(pblk); + pblk_put_global_caches(); kfree(pblk->pad_dist); } @@ -504,6 +534,9 @@ static void pblk_line_mg_free(struct pblk *pblk) pblk_mfree(l_mg->eline_meta[i]->buf, l_mg->emeta_alloc_type); kfree(l_mg->eline_meta[i]); } + + mempool_destroy(l_mg->bitmap_pool); + kmem_cache_destroy(l_mg->bitmap_cache); } static void pblk_line_meta_free(struct pblk_line_mgmt *l_mg, @@ -540,67 +573,6 @@ static void pblk_lines_free(struct pblk *pblk) kfree(pblk->lines); } -static int pblk_bb_get_tbl(struct nvm_tgt_dev *dev, struct pblk_lun *rlun, - u8 *blks, int nr_blks) -{ - struct ppa_addr ppa; - int ret; - - ppa.ppa = 0; - ppa.g.ch = rlun->bppa.g.ch; - ppa.g.lun = rlun->bppa.g.lun; - - ret = nvm_get_tgt_bb_tbl(dev, ppa, blks); - if (ret) - return ret; - - nr_blks = nvm_bb_tbl_fold(dev->parent, blks, nr_blks); - if (nr_blks < 0) - return -EIO; - - return 0; -} - -static void *pblk_bb_get_meta(struct pblk *pblk) -{ - struct nvm_tgt_dev *dev = pblk->dev; - struct nvm_geo *geo = &dev->geo; - u8 *meta; - int i, nr_blks, blk_per_lun; - int ret; - - blk_per_lun = geo->num_chk * geo->pln_mode; - nr_blks = blk_per_lun * geo->all_luns; - - meta = kmalloc(nr_blks, GFP_KERNEL); - if (!meta) - return ERR_PTR(-ENOMEM); - - for (i = 0; i < geo->all_luns; i++) { - struct pblk_lun *rlun = &pblk->luns[i]; - u8 *meta_pos = meta + i * blk_per_lun; - - ret = pblk_bb_get_tbl(dev, rlun, meta_pos, blk_per_lun); - if (ret) { - kfree(meta); - return ERR_PTR(-EIO); - } - } - - return meta; -} - -static void *pblk_chunk_get_meta(struct pblk *pblk) -{ - struct nvm_tgt_dev *dev = pblk->dev; - struct nvm_geo *geo = &dev->geo; - - if (geo->version == NVM_OCSSD_SPEC_12) - return pblk_bb_get_meta(pblk); - else - return pblk_chunk_get_info(pblk); -} - static int pblk_luns_init(struct pblk *pblk) { struct nvm_tgt_dev *dev = pblk->dev; @@ -699,51 +671,7 @@ static void pblk_set_provision(struct pblk *pblk, long nr_free_blks) atomic_set(&pblk->rl.free_user_blocks, nr_free_blks); } -static int pblk_setup_line_meta_12(struct pblk *pblk, struct pblk_line *line, - void *chunk_meta) -{ - struct nvm_tgt_dev *dev = pblk->dev; - struct nvm_geo *geo = &dev->geo; - struct pblk_line_meta *lm = &pblk->lm; - int i, chk_per_lun, nr_bad_chks = 0; - - chk_per_lun = geo->num_chk * geo->pln_mode; - - for (i = 0; i < lm->blk_per_line; i++) { - struct pblk_lun *rlun = &pblk->luns[i]; - struct nvm_chk_meta *chunk; - int pos = pblk_ppa_to_pos(geo, rlun->bppa); - u8 *lun_bb_meta = chunk_meta + pos * chk_per_lun; - - chunk = &line->chks[pos]; - - /* - * In 1.2 spec. chunk state is not persisted by the device. Thus - * some of the values are reset each time pblk is instantiated, - * so we have to assume that the block is closed. - */ - if (lun_bb_meta[line->id] == NVM_BLK_T_FREE) - chunk->state = NVM_CHK_ST_CLOSED; - else - chunk->state = NVM_CHK_ST_OFFLINE; - - chunk->type = NVM_CHK_TP_W_SEQ; - chunk->wi = 0; - chunk->slba = -1; - chunk->cnlb = geo->clba; - chunk->wp = 0; - - if (!(chunk->state & NVM_CHK_ST_OFFLINE)) - continue; - - set_bit(pos, line->blk_bitmap); - nr_bad_chks++; - } - - return nr_bad_chks; -} - -static int pblk_setup_line_meta_20(struct pblk *pblk, struct pblk_line *line, +static int pblk_setup_line_meta_chk(struct pblk *pblk, struct pblk_line *line, struct nvm_chk_meta *meta) { struct nvm_tgt_dev *dev = pblk->dev; @@ -772,6 +700,9 @@ static int pblk_setup_line_meta_20(struct pblk *pblk, struct pblk_line *line, chunk->cnlb = chunk_meta->cnlb; chunk->wp = chunk_meta->wp; + trace_pblk_chunk_state(pblk_disk_name(pblk), &ppa, + chunk->state); + if (chunk->type & NVM_CHK_TP_SZ_SPEC) { WARN_ONCE(1, "pblk: custom-sized chunks unsupported\n"); continue; @@ -790,8 +721,6 @@ static int pblk_setup_line_meta_20(struct pblk *pblk, struct pblk_line *line, static long pblk_setup_line_meta(struct pblk *pblk, struct pblk_line *line, void *chunk_meta, int line_id) { - struct nvm_tgt_dev *dev = pblk->dev; - struct nvm_geo *geo = &dev->geo; struct pblk_line_mgmt *l_mg = &pblk->l_mg; struct pblk_line_meta *lm = &pblk->lm; long nr_bad_chks, chk_in_line; @@ -804,10 +733,7 @@ static long pblk_setup_line_meta(struct pblk *pblk, struct pblk_line *line, line->vsc = &l_mg->vsc_list[line_id]; spin_lock_init(&line->lock); - if (geo->version == NVM_OCSSD_SPEC_12) - nr_bad_chks = pblk_setup_line_meta_12(pblk, line, chunk_meta); - else - nr_bad_chks = pblk_setup_line_meta_20(pblk, line, chunk_meta); + nr_bad_chks = pblk_setup_line_meta_chk(pblk, line, chunk_meta); chk_in_line = lm->blk_per_line - nr_bad_chks; if (nr_bad_chks < 0 || nr_bad_chks > lm->blk_per_line || @@ -913,6 +839,17 @@ static int pblk_line_mg_init(struct pblk *pblk) goto fail_free_smeta; } + l_mg->bitmap_cache = kmem_cache_create("pblk_lm_bitmap", + lm->sec_bitmap_len, 0, 0, NULL); + if (!l_mg->bitmap_cache) + goto fail_free_smeta; + + /* the bitmap pool is used for both valid and map bitmaps */ + l_mg->bitmap_pool = mempool_create_slab_pool(PBLK_DATA_LINES * 2, + l_mg->bitmap_cache); + if (!l_mg->bitmap_pool) + goto fail_destroy_bitmap_cache; + /* emeta allocates three different buffers for managing metadata with * in-memory and in-media layouts */ @@ -965,6 +902,10 @@ fail_free_emeta: kfree(l_mg->eline_meta[i]->buf); kfree(l_mg->eline_meta[i]); } + + mempool_destroy(l_mg->bitmap_pool); +fail_destroy_bitmap_cache: + kmem_cache_destroy(l_mg->bitmap_cache); fail_free_smeta: for (i = 0; i < PBLK_DATA_LINES; i++) kfree(l_mg->sline_meta[i]); @@ -1058,7 +999,7 @@ static int pblk_lines_init(struct pblk *pblk) if (ret) goto fail_free_meta; - chunk_meta = pblk_chunk_get_meta(pblk); + chunk_meta = pblk_get_chunk_meta(pblk); if (IS_ERR(chunk_meta)) { ret = PTR_ERR(chunk_meta); goto fail_free_luns; @@ -1079,16 +1020,20 @@ static int pblk_lines_init(struct pblk *pblk) goto fail_free_lines; nr_free_chks += pblk_setup_line_meta(pblk, line, chunk_meta, i); + + trace_pblk_line_state(pblk_disk_name(pblk), line->id, + line->state); } if (!nr_free_chks) { pblk_err(pblk, "too many bad blocks prevent for sane instance\n"); - return -EINTR; + ret = -EINTR; + goto fail_free_lines; } pblk_set_provision(pblk, nr_free_chks); - kfree(chunk_meta); + vfree(chunk_meta); return 0; fail_free_lines: @@ -1165,7 +1110,6 @@ static void pblk_exit(void *private, bool graceful) { struct pblk *pblk = private; - down_write(&pblk_lock); pblk_gc_exit(pblk, graceful); pblk_tear_down(pblk, graceful); @@ -1174,7 +1118,6 @@ static void pblk_exit(void *private, bool graceful) #endif pblk_free(pblk); - up_write(&pblk_lock); } static sector_t pblk_capacity(void *private) @@ -1200,6 +1143,7 @@ static void *pblk_init(struct nvm_tgt_dev *dev, struct gendisk *tdisk, pblk->dev = dev; pblk->disk = tdisk; pblk->state = PBLK_STATE_RUNNING; + trace_pblk_state(pblk_disk_name(pblk), pblk->state); pblk->gc.gc_enabled = 0; if (!(geo->version == NVM_OCSSD_SPEC_12 || @@ -1210,13 +1154,6 @@ static void *pblk_init(struct nvm_tgt_dev *dev, struct gendisk *tdisk, return ERR_PTR(-EINVAL); } - if (geo->version == NVM_OCSSD_SPEC_12 && geo->dom & NVM_RSP_L2P) { - pblk_err(pblk, "host-side L2P table not supported. (%x)\n", - geo->dom); - kfree(pblk); - return ERR_PTR(-EINVAL); - } - spin_lock_init(&pblk->resubmit_lock); spin_lock_init(&pblk->trans_lock); spin_lock_init(&pblk->lock); diff --git a/drivers/lightnvm/pblk-map.c b/drivers/lightnvm/pblk-map.c index 953ca31dda68..6dcbd44e3acb 100644 --- a/drivers/lightnvm/pblk-map.c +++ b/drivers/lightnvm/pblk-map.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2016 CNEX Labs * Initial release: Javier Gonzalez <javier@cnexlabs.com> @@ -79,7 +80,7 @@ static int pblk_map_page_data(struct pblk *pblk, unsigned int sentry, } } - pblk_down_rq(pblk, ppa_list, nr_secs, lun_bitmap); + pblk_down_rq(pblk, ppa_list[0], lun_bitmap); return 0; } @@ -88,13 +89,14 @@ void pblk_map_rq(struct pblk *pblk, struct nvm_rq *rqd, unsigned int sentry, unsigned int off) { struct pblk_sec_meta *meta_list = rqd->meta_list; + struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); unsigned int map_secs; int min = pblk->min_write_pgs; int i; for (i = off; i < rqd->nr_ppas; i += min) { map_secs = (i + min > valid_secs) ? (valid_secs % min) : min; - if (pblk_map_page_data(pblk, sentry + i, &rqd->ppa_list[i], + if (pblk_map_page_data(pblk, sentry + i, &ppa_list[i], lun_bitmap, &meta_list[i], map_secs)) { bio_put(rqd->bio); pblk_free_rqd(pblk, rqd, PBLK_WRITE); @@ -112,6 +114,7 @@ void pblk_map_erase_rq(struct pblk *pblk, struct nvm_rq *rqd, struct nvm_geo *geo = &dev->geo; struct pblk_line_meta *lm = &pblk->lm; struct pblk_sec_meta *meta_list = rqd->meta_list; + struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); struct pblk_line *e_line, *d_line; unsigned int map_secs; int min = pblk->min_write_pgs; @@ -119,14 +122,14 @@ void pblk_map_erase_rq(struct pblk *pblk, struct nvm_rq *rqd, for (i = 0; i < rqd->nr_ppas; i += min) { map_secs = (i + min > valid_secs) ? (valid_secs % min) : min; - if (pblk_map_page_data(pblk, sentry + i, &rqd->ppa_list[i], + if (pblk_map_page_data(pblk, sentry + i, &ppa_list[i], lun_bitmap, &meta_list[i], map_secs)) { bio_put(rqd->bio); pblk_free_rqd(pblk, rqd, PBLK_WRITE); pblk_pipeline_stop(pblk); } - erase_lun = pblk_ppa_to_pos(geo, rqd->ppa_list[i]); + erase_lun = pblk_ppa_to_pos(geo, ppa_list[i]); /* line can change after page map. We might also be writing the * last line. @@ -141,7 +144,7 @@ void pblk_map_erase_rq(struct pblk *pblk, struct nvm_rq *rqd, set_bit(erase_lun, e_line->erase_bitmap); atomic_dec(&e_line->left_eblks); - *erase_ppa = rqd->ppa_list[i]; + *erase_ppa = ppa_list[i]; erase_ppa->a.blk = e_line->id; spin_unlock(&e_line->lock); diff --git a/drivers/lightnvm/pblk-rb.c b/drivers/lightnvm/pblk-rb.c index f6eec0212dfc..b1f4b51783f4 100644 --- a/drivers/lightnvm/pblk-rb.c +++ b/drivers/lightnvm/pblk-rb.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2016 CNEX Labs * Initial release: Javier Gonzalez <javier@cnexlabs.com> @@ -22,7 +23,7 @@ static DECLARE_RWSEM(pblk_rb_lock); -void pblk_rb_data_free(struct pblk_rb *rb) +static void pblk_rb_data_free(struct pblk_rb *rb) { struct pblk_rb_pages *p, *t; @@ -35,25 +36,51 @@ void pblk_rb_data_free(struct pblk_rb *rb) up_write(&pblk_rb_lock); } +void pblk_rb_free(struct pblk_rb *rb) +{ + pblk_rb_data_free(rb); + vfree(rb->entries); +} + +/* + * pblk_rb_calculate_size -- calculate the size of the write buffer + */ +static unsigned int pblk_rb_calculate_size(unsigned int nr_entries) +{ + /* Alloc a write buffer that can at least fit 128 entries */ + return (1 << max(get_count_order(nr_entries), 7)); +} + /* * Initialize ring buffer. The data and metadata buffers must be previously * allocated and their size must be a power of two * (Documentation/core-api/circular-buffers.rst) */ -int pblk_rb_init(struct pblk_rb *rb, struct pblk_rb_entry *rb_entry_base, - unsigned int power_size, unsigned int power_seg_sz) +int pblk_rb_init(struct pblk_rb *rb, unsigned int size, unsigned int threshold, + unsigned int seg_size) { struct pblk *pblk = container_of(rb, struct pblk, rwb); + struct pblk_rb_entry *entries; unsigned int init_entry = 0; - unsigned int alloc_order = power_size; unsigned int max_order = MAX_ORDER - 1; - unsigned int order, iter; + unsigned int power_size, power_seg_sz; + unsigned int alloc_order, order, iter; + unsigned int nr_entries; + + nr_entries = pblk_rb_calculate_size(size); + entries = vzalloc(array_size(nr_entries, sizeof(struct pblk_rb_entry))); + if (!entries) + return -ENOMEM; + + power_size = get_count_order(size); + power_seg_sz = get_count_order(seg_size); down_write(&pblk_rb_lock); - rb->entries = rb_entry_base; + rb->entries = entries; rb->seg_size = (1 << power_seg_sz); rb->nr_entries = (1 << power_size); rb->mem = rb->subm = rb->sync = rb->l2p_update = 0; + rb->back_thres = threshold; rb->flush_point = EMPTY_ENTRY; spin_lock_init(&rb->w_lock); @@ -61,6 +88,7 @@ int pblk_rb_init(struct pblk_rb *rb, struct pblk_rb_entry *rb_entry_base, INIT_LIST_HEAD(&rb->pages); + alloc_order = power_size; if (alloc_order >= max_order) { order = max_order; iter = (1 << (alloc_order - max_order)); @@ -79,6 +107,7 @@ int pblk_rb_init(struct pblk_rb *rb, struct pblk_rb_entry *rb_entry_base, page_set = kmalloc(sizeof(struct pblk_rb_pages), GFP_KERNEL); if (!page_set) { up_write(&pblk_rb_lock); + vfree(entries); return -ENOMEM; } @@ -88,6 +117,7 @@ int pblk_rb_init(struct pblk_rb *rb, struct pblk_rb_entry *rb_entry_base, kfree(page_set); pblk_rb_data_free(rb); up_write(&pblk_rb_lock); + vfree(entries); return -ENOMEM; } kaddr = page_address(page_set->pages); @@ -124,20 +154,6 @@ int pblk_rb_init(struct pblk_rb *rb, struct pblk_rb_entry *rb_entry_base, return 0; } -/* - * pblk_rb_calculate_size -- calculate the size of the write buffer - */ -unsigned int pblk_rb_calculate_size(unsigned int nr_entries) -{ - /* Alloc a write buffer that can at least fit 128 entries */ - return (1 << max(get_count_order(nr_entries), 7)); -} - -void *pblk_rb_entries_ref(struct pblk_rb *rb) -{ - return rb->entries; -} - static void clean_wctx(struct pblk_w_ctx *w_ctx) { int flags; @@ -168,6 +184,12 @@ static unsigned int pblk_rb_space(struct pblk_rb *rb) return pblk_rb_ring_space(rb, mem, sync, rb->nr_entries); } +unsigned int pblk_rb_ptr_wrap(struct pblk_rb *rb, unsigned int p, + unsigned int nr_entries) +{ + return (p + nr_entries) & (rb->nr_entries - 1); +} + /* * Buffer count is calculated with respect to the submission entry signaling the * entries that are available to send to the media @@ -194,8 +216,7 @@ unsigned int pblk_rb_read_commit(struct pblk_rb *rb, unsigned int nr_entries) subm = READ_ONCE(rb->subm); /* Commit read means updating submission pointer */ - smp_store_release(&rb->subm, - (subm + nr_entries) & (rb->nr_entries - 1)); + smp_store_release(&rb->subm, pblk_rb_ptr_wrap(rb, subm, nr_entries)); return subm; } @@ -225,10 +246,10 @@ static int __pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int to_update) pblk_update_map_dev(pblk, w_ctx->lba, w_ctx->ppa, entry->cacheline); - line = &pblk->lines[pblk_ppa_to_line(w_ctx->ppa)]; + line = pblk_ppa_to_line(pblk, w_ctx->ppa); kref_put(&line->ref, pblk_line_put); clean_wctx(w_ctx); - rb->l2p_update = (rb->l2p_update + 1) & (rb->nr_entries - 1); + rb->l2p_update = pblk_rb_ptr_wrap(rb, rb->l2p_update, 1); } pblk_rl_out(&pblk->rl, user_io, gc_io); @@ -385,11 +406,14 @@ static int __pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries, { unsigned int mem; unsigned int sync; + unsigned int threshold; sync = READ_ONCE(rb->sync); mem = READ_ONCE(rb->mem); - if (pblk_rb_ring_space(rb, mem, sync, rb->nr_entries) < nr_entries) + threshold = nr_entries + rb->back_thres; + + if (pblk_rb_ring_space(rb, mem, sync, rb->nr_entries) < threshold) return 0; if (pblk_rb_update_l2p(rb, nr_entries, mem, sync)) @@ -407,7 +431,7 @@ static int pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries, return 0; /* Protect from read count */ - smp_store_release(&rb->mem, (*pos + nr_entries) & (rb->nr_entries - 1)); + smp_store_release(&rb->mem, pblk_rb_ptr_wrap(rb, *pos, nr_entries)); return 1; } @@ -431,7 +455,7 @@ static int pblk_rb_may_write_flush(struct pblk_rb *rb, unsigned int nr_entries, if (!__pblk_rb_may_write(rb, nr_entries, pos)) return 0; - mem = (*pos + nr_entries) & (rb->nr_entries - 1); + mem = pblk_rb_ptr_wrap(rb, *pos, nr_entries); *io_ret = NVM_IO_DONE; if (bio->bi_opf & REQ_PREFLUSH) { @@ -571,7 +595,7 @@ try: /* Release flags on context. Protect from writes */ smp_store_release(&entry->w_ctx.flags, flags); - pos = (pos + 1) & (rb->nr_entries - 1); + pos = pblk_rb_ptr_wrap(rb, pos, 1); } if (pad) { @@ -651,7 +675,7 @@ out: struct pblk_w_ctx *pblk_rb_w_ctx(struct pblk_rb *rb, unsigned int pos) { - unsigned int entry = pos & (rb->nr_entries - 1); + unsigned int entry = pblk_rb_ptr_wrap(rb, pos, 0); return &rb->entries[entry].w_ctx; } @@ -697,7 +721,7 @@ unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries) } } - sync = (sync + nr_entries) & (rb->nr_entries - 1); + sync = pblk_rb_ptr_wrap(rb, sync, nr_entries); /* Protect from counts */ smp_store_release(&rb->sync, sync); @@ -728,32 +752,6 @@ unsigned int pblk_rb_flush_point_count(struct pblk_rb *rb) return (submitted < to_flush) ? (to_flush - submitted) : 0; } -/* - * Scan from the current position of the sync pointer to find the entry that - * corresponds to the given ppa. This is necessary since write requests can be - * completed out of order. The assumption is that the ppa is close to the sync - * pointer thus the search will not take long. - * - * The caller of this function must guarantee that the sync pointer will no - * reach the entry while it is using the metadata associated with it. With this - * assumption in mind, there is no need to take the sync lock. - */ -struct pblk_rb_entry *pblk_rb_sync_scan_entry(struct pblk_rb *rb, - struct ppa_addr *ppa) -{ - unsigned int sync, subm, count; - unsigned int i; - - sync = READ_ONCE(rb->sync); - subm = READ_ONCE(rb->subm); - count = pblk_rb_ring_count(subm, sync, rb->nr_entries); - - for (i = 0; i < count; i++) - sync = (sync + 1) & (rb->nr_entries - 1); - - return NULL; -} - int pblk_rb_tear_down_check(struct pblk_rb *rb) { struct pblk_rb_entry *entry; diff --git a/drivers/lightnvm/pblk-read.c b/drivers/lightnvm/pblk-read.c index 5a46d7f9302f..9fba614adeeb 100644 --- a/drivers/lightnvm/pblk-read.c +++ b/drivers/lightnvm/pblk-read.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2016 CNEX Labs * Initial release: Javier Gonzalez <javier@cnexlabs.com> @@ -43,7 +44,7 @@ static void pblk_read_ppalist_rq(struct pblk *pblk, struct nvm_rq *rqd, unsigned long *read_bitmap) { struct pblk_sec_meta *meta_list = rqd->meta_list; - struct ppa_addr ppas[PBLK_MAX_REQ_ADDRS]; + struct ppa_addr ppas[NVM_MAX_VLBA]; int nr_secs = rqd->nr_ppas; bool advanced_bio = false; int i, j = 0; @@ -93,9 +94,7 @@ next: } if (pblk_io_aligned(pblk, nr_secs)) - rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_SEQUENTIAL); - else - rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM); + rqd->is_seq = 1; #ifdef CONFIG_NVM_PBLK_DEBUG atomic_long_add(nr_secs, &pblk->inflight_reads); @@ -118,10 +117,9 @@ static void pblk_read_check_seq(struct pblk *pblk, struct nvm_rq *rqd, if (lba != blba + i) { #ifdef CONFIG_NVM_PBLK_DEBUG - struct ppa_addr *p; + struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); - p = (nr_lbas == 1) ? &rqd->ppa_list[i] : &rqd->ppa_addr; - print_ppa(pblk, p, "seq", i); + print_ppa(pblk, &ppa_list[i], "seq", i); #endif pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n", lba, (u64)blba + i); @@ -150,14 +148,12 @@ static void pblk_read_check_rand(struct pblk *pblk, struct nvm_rq *rqd, if (lba != meta_lba) { #ifdef CONFIG_NVM_PBLK_DEBUG - struct ppa_addr *p; - int nr_ppas = rqd->nr_ppas; + struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); - p = (nr_ppas == 1) ? &rqd->ppa_list[j] : &rqd->ppa_addr; - print_ppa(pblk, p, "seq", j); + print_ppa(pblk, &ppa_list[j], "rnd", j); #endif pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n", - lba, meta_lba); + meta_lba, lba); WARN_ON(1); } @@ -167,22 +163,6 @@ static void pblk_read_check_rand(struct pblk *pblk, struct nvm_rq *rqd, WARN_ONCE(j != rqd->nr_ppas, "pblk: corrupted random request\n"); } -static void pblk_read_put_rqd_kref(struct pblk *pblk, struct nvm_rq *rqd) -{ - struct ppa_addr *ppa_list; - int i; - - ppa_list = (rqd->nr_ppas > 1) ? rqd->ppa_list : &rqd->ppa_addr; - - for (i = 0; i < rqd->nr_ppas; i++) { - struct ppa_addr ppa = ppa_list[i]; - struct pblk_line *line; - - line = &pblk->lines[pblk_ppa_to_line(ppa)]; - kref_put(&line->ref, pblk_line_put_wq); - } -} - static void pblk_end_user_read(struct bio *bio) { #ifdef CONFIG_NVM_PBLK_DEBUG @@ -210,7 +190,7 @@ static void __pblk_end_io_read(struct pblk *pblk, struct nvm_rq *rqd, bio_put(int_bio); if (put_line) - pblk_read_put_rqd_kref(pblk, rqd); + pblk_rq_to_line_put(pblk, rqd); #ifdef CONFIG_NVM_PBLK_DEBUG atomic_long_add(rqd->nr_ppas, &pblk->sync_reads); @@ -270,9 +250,9 @@ static void pblk_end_partial_read(struct nvm_rq *rqd) i = 0; hole = find_first_zero_bit(read_bitmap, nr_secs); do { - int line_id = pblk_ppa_to_line(rqd->ppa_list[i]); - struct pblk_line *line = &pblk->lines[line_id]; + struct pblk_line *line; + line = pblk_ppa_to_line(pblk, rqd->ppa_list[i]); kref_put(&line->ref, pblk_line_put); meta_list[hole].lba = lba_list_media[i]; @@ -344,7 +324,6 @@ static int pblk_setup_partial_read(struct pblk *pblk, struct nvm_rq *rqd, rqd->bio = new_bio; rqd->nr_ppas = nr_holes; - rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM); pr_ctx->ppa_ptr = NULL; pr_ctx->orig_bio = bio; @@ -438,8 +417,6 @@ retry: } else { rqd->ppa_addr = ppa; } - - rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM); } int pblk_submit_read(struct pblk *pblk, struct bio *bio) @@ -454,13 +431,6 @@ int pblk_submit_read(struct pblk *pblk, struct bio *bio) DECLARE_BITMAP(read_bitmap, NVM_MAX_VLBA); int ret = NVM_IO_ERR; - /* logic error: lba out-of-bounds. Ignore read request */ - if (blba >= pblk->rl.nr_secs || nr_secs > PBLK_MAX_REQ_ADDRS) { - WARN(1, "pblk: read lba out of bounds (lba:%llu, nr:%d)\n", - (unsigned long long)blba, nr_secs); - return NVM_IO_ERR; - } - generic_start_io_acct(q, REQ_OP_READ, bio_sectors(bio), &pblk->disk->part0); @@ -484,21 +454,13 @@ int pblk_submit_read(struct pblk *pblk, struct bio *bio) */ bio_init_idx = pblk_get_bi_idx(bio); - rqd->meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, - &rqd->dma_meta_list); - if (!rqd->meta_list) { - pblk_err(pblk, "not able to allocate ppa list\n"); + if (pblk_alloc_rqd_meta(pblk, rqd)) goto fail_rqd_free; - } - - if (nr_secs > 1) { - rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size; - rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size; + if (nr_secs > 1) pblk_read_ppalist_rq(pblk, rqd, bio, blba, read_bitmap); - } else { + else pblk_read_rq(pblk, rqd, bio, blba, read_bitmap); - } if (bitmap_full(read_bitmap, nr_secs)) { atomic_inc(&pblk->inflight_io); @@ -552,7 +514,7 @@ static int read_ppalist_rq_gc(struct pblk *pblk, struct nvm_rq *rqd, struct pblk_line *line, u64 *lba_list, u64 *paddr_list_gc, unsigned int nr_secs) { - struct ppa_addr ppa_list_l2p[PBLK_MAX_REQ_ADDRS]; + struct ppa_addr ppa_list_l2p[NVM_MAX_VLBA]; struct ppa_addr ppa_gc; int valid_secs = 0; int i; @@ -625,15 +587,11 @@ int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq) memset(&rqd, 0, sizeof(struct nvm_rq)); - rqd.meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, - &rqd.dma_meta_list); - if (!rqd.meta_list) - return -ENOMEM; + ret = pblk_alloc_rqd_meta(pblk, &rqd); + if (ret) + return ret; if (gc_rq->nr_secs > 1) { - rqd.ppa_list = rqd.meta_list + pblk_dma_meta_size; - rqd.dma_ppa_list = rqd.dma_meta_list + pblk_dma_meta_size; - gc_rq->secs_to_gc = read_ppalist_rq_gc(pblk, &rqd, gc_rq->line, gc_rq->lba_list, gc_rq->paddr_list, @@ -654,7 +612,8 @@ int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq) PBLK_VMALLOC_META, GFP_KERNEL); if (IS_ERR(bio)) { pblk_err(pblk, "could not allocate GC bio (%lu)\n", - PTR_ERR(bio)); + PTR_ERR(bio)); + ret = PTR_ERR(bio); goto err_free_dma; } @@ -663,7 +622,6 @@ int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq) rqd.opcode = NVM_OP_PREAD; rqd.nr_ppas = gc_rq->secs_to_gc; - rqd.flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM); rqd.bio = bio; if (pblk_submit_io_sync(pblk, &rqd)) { @@ -690,12 +648,12 @@ int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq) #endif out: - nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list); + pblk_free_rqd_meta(pblk, &rqd); return ret; err_free_bio: bio_put(bio); err_free_dma: - nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list); + pblk_free_rqd_meta(pblk, &rqd); return ret; } diff --git a/drivers/lightnvm/pblk-recovery.c b/drivers/lightnvm/pblk-recovery.c index e232e47e1353..5740b7509bd8 100644 --- a/drivers/lightnvm/pblk-recovery.c +++ b/drivers/lightnvm/pblk-recovery.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2016 CNEX Labs * Initial: Javier Gonzalez <javier@cnexlabs.com> @@ -15,6 +16,7 @@ */ #include "pblk.h" +#include "pblk-trace.h" int pblk_recov_check_emeta(struct pblk *pblk, struct line_emeta *emeta_buf) { @@ -85,15 +87,39 @@ static int pblk_recov_l2p_from_emeta(struct pblk *pblk, struct pblk_line *line) return 0; } -static int pblk_calc_sec_in_line(struct pblk *pblk, struct pblk_line *line) +static void pblk_update_line_wp(struct pblk *pblk, struct pblk_line *line, + u64 written_secs) +{ + int i; + + for (i = 0; i < written_secs; i += pblk->min_write_pgs) + pblk_alloc_page(pblk, line, pblk->min_write_pgs); +} + +static u64 pblk_sec_in_open_line(struct pblk *pblk, struct pblk_line *line) { - struct nvm_tgt_dev *dev = pblk->dev; - struct nvm_geo *geo = &dev->geo; struct pblk_line_meta *lm = &pblk->lm; int nr_bb = bitmap_weight(line->blk_bitmap, lm->blk_per_line); + u64 written_secs = 0; + int valid_chunks = 0; + int i; + + for (i = 0; i < lm->blk_per_line; i++) { + struct nvm_chk_meta *chunk = &line->chks[i]; + + if (chunk->state & NVM_CHK_ST_OFFLINE) + continue; + + written_secs += chunk->wp; + valid_chunks++; + } + + if (lm->blk_per_line - nr_bb != valid_chunks) + pblk_err(pblk, "recovery line %d is bad\n", line->id); - return lm->sec_per_line - lm->smeta_sec - lm->emeta_sec[0] - - nr_bb * geo->clba; + pblk_update_line_wp(pblk, line, written_secs - lm->smeta_sec); + + return written_secs; } struct pblk_recov_alloc { @@ -105,115 +131,6 @@ struct pblk_recov_alloc { dma_addr_t dma_meta_list; }; -static int pblk_recov_read_oob(struct pblk *pblk, struct pblk_line *line, - struct pblk_recov_alloc p, u64 r_ptr) -{ - struct nvm_tgt_dev *dev = pblk->dev; - struct nvm_geo *geo = &dev->geo; - struct ppa_addr *ppa_list; - struct pblk_sec_meta *meta_list; - struct nvm_rq *rqd; - struct bio *bio; - void *data; - dma_addr_t dma_ppa_list, dma_meta_list; - u64 r_ptr_int; - int left_ppas; - int rq_ppas, rq_len; - int i, j; - int ret = 0; - - ppa_list = p.ppa_list; - meta_list = p.meta_list; - rqd = p.rqd; - data = p.data; - dma_ppa_list = p.dma_ppa_list; - dma_meta_list = p.dma_meta_list; - - left_ppas = line->cur_sec - r_ptr; - if (!left_ppas) - return 0; - - r_ptr_int = r_ptr; - -next_read_rq: - memset(rqd, 0, pblk_g_rq_size); - - rq_ppas = pblk_calc_secs(pblk, left_ppas, 0); - if (!rq_ppas) - rq_ppas = pblk->min_write_pgs; - rq_len = rq_ppas * geo->csecs; - - bio = bio_map_kern(dev->q, data, rq_len, GFP_KERNEL); - if (IS_ERR(bio)) - return PTR_ERR(bio); - - bio->bi_iter.bi_sector = 0; /* internal bio */ - bio_set_op_attrs(bio, REQ_OP_READ, 0); - - rqd->bio = bio; - rqd->opcode = NVM_OP_PREAD; - rqd->meta_list = meta_list; - rqd->nr_ppas = rq_ppas; - rqd->ppa_list = ppa_list; - rqd->dma_ppa_list = dma_ppa_list; - rqd->dma_meta_list = dma_meta_list; - - if (pblk_io_aligned(pblk, rq_ppas)) - rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_SEQUENTIAL); - else - rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM); - - for (i = 0; i < rqd->nr_ppas; ) { - struct ppa_addr ppa; - int pos; - - ppa = addr_to_gen_ppa(pblk, r_ptr_int, line->id); - pos = pblk_ppa_to_pos(geo, ppa); - - while (test_bit(pos, line->blk_bitmap)) { - r_ptr_int += pblk->min_write_pgs; - ppa = addr_to_gen_ppa(pblk, r_ptr_int, line->id); - pos = pblk_ppa_to_pos(geo, ppa); - } - - for (j = 0; j < pblk->min_write_pgs; j++, i++, r_ptr_int++) - rqd->ppa_list[i] = - addr_to_gen_ppa(pblk, r_ptr_int, line->id); - } - - /* If read fails, more padding is needed */ - ret = pblk_submit_io_sync(pblk, rqd); - if (ret) { - pblk_err(pblk, "I/O submission failed: %d\n", ret); - return ret; - } - - atomic_dec(&pblk->inflight_io); - - /* At this point, the read should not fail. If it does, it is a problem - * we cannot recover from here. Need FTL log. - */ - if (rqd->error && rqd->error != NVM_RSP_WARN_HIGHECC) { - pblk_err(pblk, "L2P recovery failed (%d)\n", rqd->error); - return -EINTR; - } - - for (i = 0; i < rqd->nr_ppas; i++) { - u64 lba = le64_to_cpu(meta_list[i].lba); - - if (lba == ADDR_EMPTY || lba > pblk->rl.nr_secs) - continue; - - pblk_update_map(pblk, lba, rqd->ppa_list[i]); - } - - left_ppas -= rq_ppas; - if (left_ppas > 0) - goto next_read_rq; - - return 0; -} - static void pblk_recov_complete(struct kref *ref) { struct pblk_pad_rq *pad_rq = container_of(ref, struct pblk_pad_rq, ref); @@ -223,10 +140,11 @@ static void pblk_recov_complete(struct kref *ref) static void pblk_end_io_recov(struct nvm_rq *rqd) { + struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); struct pblk_pad_rq *pad_rq = rqd->private; struct pblk *pblk = pad_rq->pblk; - pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas); + pblk_up_chunk(pblk, ppa_list[0]); pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT); @@ -234,18 +152,17 @@ static void pblk_end_io_recov(struct nvm_rq *rqd) kref_put(&pad_rq->ref, pblk_recov_complete); } -static int pblk_recov_pad_oob(struct pblk *pblk, struct pblk_line *line, - int left_ppas) +/* pad line using line bitmap. */ +static int pblk_recov_pad_line(struct pblk *pblk, struct pblk_line *line, + int left_ppas) { struct nvm_tgt_dev *dev = pblk->dev; struct nvm_geo *geo = &dev->geo; - struct ppa_addr *ppa_list; struct pblk_sec_meta *meta_list; struct pblk_pad_rq *pad_rq; struct nvm_rq *rqd; struct bio *bio; void *data; - dma_addr_t dma_ppa_list, dma_meta_list; __le64 *lba_list = emeta_to_lbas(pblk, line->emeta->buf); u64 w_ptr = line->cur_sec; int left_line_ppas, rq_ppas, rq_len; @@ -279,20 +196,11 @@ next_pad_rq: rq_len = rq_ppas * geo->csecs; - meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, &dma_meta_list); - if (!meta_list) { - ret = -ENOMEM; - goto fail_free_pad; - } - - ppa_list = (void *)(meta_list) + pblk_dma_meta_size; - dma_ppa_list = dma_meta_list + pblk_dma_meta_size; - bio = pblk_bio_map_addr(pblk, data, rq_ppas, rq_len, PBLK_VMALLOC_META, GFP_KERNEL); if (IS_ERR(bio)) { ret = PTR_ERR(bio); - goto fail_free_meta; + goto fail_free_pad; } bio->bi_iter.bi_sector = 0; /* internal bio */ @@ -300,17 +208,19 @@ next_pad_rq: rqd = pblk_alloc_rqd(pblk, PBLK_WRITE_INT); + ret = pblk_alloc_rqd_meta(pblk, rqd); + if (ret) + goto fail_free_rqd; + rqd->bio = bio; rqd->opcode = NVM_OP_PWRITE; - rqd->flags = pblk_set_progr_mode(pblk, PBLK_WRITE); - rqd->meta_list = meta_list; + rqd->is_seq = 1; rqd->nr_ppas = rq_ppas; - rqd->ppa_list = ppa_list; - rqd->dma_ppa_list = dma_ppa_list; - rqd->dma_meta_list = dma_meta_list; rqd->end_io = pblk_end_io_recov; rqd->private = pad_rq; + meta_list = rqd->meta_list; + for (i = 0; i < rqd->nr_ppas; ) { struct ppa_addr ppa; int pos; @@ -338,13 +248,13 @@ next_pad_rq: } kref_get(&pad_rq->ref); - pblk_down_page(pblk, rqd->ppa_list, rqd->nr_ppas); + pblk_down_chunk(pblk, rqd->ppa_list[0]); ret = pblk_submit_io(pblk, rqd); if (ret) { pblk_err(pblk, "I/O submission failed: %d\n", ret); - pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas); - goto fail_free_bio; + pblk_up_chunk(pblk, rqd->ppa_list[0]); + goto fail_free_rqd; } left_line_ppas -= rq_ppas; @@ -368,157 +278,60 @@ free_rq: kfree(pad_rq); return ret; -fail_free_bio: +fail_free_rqd: + pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT); bio_put(bio); -fail_free_meta: - nvm_dev_dma_free(dev->parent, meta_list, dma_meta_list); fail_free_pad: kfree(pad_rq); vfree(data); return ret; } -/* When this function is called, it means that not all upper pages have been - * written in a page that contains valid data. In order to recover this data, we - * first find the write pointer on the device, then we pad all necessary - * sectors, and finally attempt to read the valid data - */ -static int pblk_recov_scan_all_oob(struct pblk *pblk, struct pblk_line *line, - struct pblk_recov_alloc p) +static int pblk_pad_distance(struct pblk *pblk, struct pblk_line *line) { struct nvm_tgt_dev *dev = pblk->dev; struct nvm_geo *geo = &dev->geo; - struct ppa_addr *ppa_list; - struct pblk_sec_meta *meta_list; - struct nvm_rq *rqd; - struct bio *bio; - void *data; - dma_addr_t dma_ppa_list, dma_meta_list; - u64 w_ptr = 0, r_ptr; - int rq_ppas, rq_len; - int i, j; - int ret = 0; - int rec_round; - int left_ppas = pblk_calc_sec_in_line(pblk, line) - line->cur_sec; - - ppa_list = p.ppa_list; - meta_list = p.meta_list; - rqd = p.rqd; - data = p.data; - dma_ppa_list = p.dma_ppa_list; - dma_meta_list = p.dma_meta_list; - - /* we could recover up until the line write pointer */ - r_ptr = line->cur_sec; - rec_round = 0; - -next_rq: - memset(rqd, 0, pblk_g_rq_size); + int distance = geo->mw_cunits * geo->all_luns * geo->ws_opt; - rq_ppas = pblk_calc_secs(pblk, left_ppas, 0); - if (!rq_ppas) - rq_ppas = pblk->min_write_pgs; - rq_len = rq_ppas * geo->csecs; - - bio = bio_map_kern(dev->q, data, rq_len, GFP_KERNEL); - if (IS_ERR(bio)) - return PTR_ERR(bio); - - bio->bi_iter.bi_sector = 0; /* internal bio */ - bio_set_op_attrs(bio, REQ_OP_READ, 0); + return (distance > line->left_msecs) ? line->left_msecs : distance; +} - rqd->bio = bio; - rqd->opcode = NVM_OP_PREAD; - rqd->meta_list = meta_list; - rqd->nr_ppas = rq_ppas; - rqd->ppa_list = ppa_list; - rqd->dma_ppa_list = dma_ppa_list; - rqd->dma_meta_list = dma_meta_list; +static int pblk_line_wp_is_unbalanced(struct pblk *pblk, + struct pblk_line *line) +{ + struct nvm_tgt_dev *dev = pblk->dev; + struct nvm_geo *geo = &dev->geo; + struct pblk_line_meta *lm = &pblk->lm; + struct pblk_lun *rlun; + struct nvm_chk_meta *chunk; + struct ppa_addr ppa; + u64 line_wp; + int pos, i; - if (pblk_io_aligned(pblk, rq_ppas)) - rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_SEQUENTIAL); - else - rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM); + rlun = &pblk->luns[0]; + ppa = rlun->bppa; + pos = pblk_ppa_to_pos(geo, ppa); + chunk = &line->chks[pos]; - for (i = 0; i < rqd->nr_ppas; ) { - struct ppa_addr ppa; - int pos; + line_wp = chunk->wp; - w_ptr = pblk_alloc_page(pblk, line, pblk->min_write_pgs); - ppa = addr_to_gen_ppa(pblk, w_ptr, line->id); + for (i = 1; i < lm->blk_per_line; i++) { + rlun = &pblk->luns[i]; + ppa = rlun->bppa; pos = pblk_ppa_to_pos(geo, ppa); + chunk = &line->chks[pos]; - while (test_bit(pos, line->blk_bitmap)) { - w_ptr += pblk->min_write_pgs; - ppa = addr_to_gen_ppa(pblk, w_ptr, line->id); - pos = pblk_ppa_to_pos(geo, ppa); - } - - for (j = 0; j < pblk->min_write_pgs; j++, i++, w_ptr++) - rqd->ppa_list[i] = - addr_to_gen_ppa(pblk, w_ptr, line->id); - } - - ret = pblk_submit_io_sync(pblk, rqd); - if (ret) { - pblk_err(pblk, "I/O submission failed: %d\n", ret); - return ret; - } - - atomic_dec(&pblk->inflight_io); - - /* This should not happen since the read failed during normal recovery, - * but the media works funny sometimes... - */ - if (!rec_round++ && !rqd->error) { - rec_round = 0; - for (i = 0; i < rqd->nr_ppas; i++, r_ptr++) { - u64 lba = le64_to_cpu(meta_list[i].lba); - - if (lba == ADDR_EMPTY || lba > pblk->rl.nr_secs) - continue; - - pblk_update_map(pblk, lba, rqd->ppa_list[i]); - } - } - - /* Reached the end of the written line */ - if (rqd->error == NVM_RSP_ERR_EMPTYPAGE) { - int pad_secs, nr_error_bits, bit; - int ret; - - bit = find_first_bit((void *)&rqd->ppa_status, rqd->nr_ppas); - nr_error_bits = rqd->nr_ppas - bit; - - /* Roll back failed sectors */ - line->cur_sec -= nr_error_bits; - line->left_msecs += nr_error_bits; - bitmap_clear(line->map_bitmap, line->cur_sec, nr_error_bits); - - pad_secs = pblk_pad_distance(pblk); - if (pad_secs > line->left_msecs) - pad_secs = line->left_msecs; - - ret = pblk_recov_pad_oob(pblk, line, pad_secs); - if (ret) - pblk_err(pblk, "OOB padding failed (err:%d)\n", ret); - - ret = pblk_recov_read_oob(pblk, line, p, r_ptr); - if (ret) - pblk_err(pblk, "OOB read failed (err:%d)\n", ret); - - left_ppas = 0; + if (chunk->wp > line_wp) + return 1; + else if (chunk->wp < line_wp) + line_wp = chunk->wp; } - left_ppas -= rq_ppas; - if (left_ppas > 0) - goto next_rq; - - return ret; + return 0; } static int pblk_recov_scan_oob(struct pblk *pblk, struct pblk_line *line, - struct pblk_recov_alloc p, int *done) + struct pblk_recov_alloc p) { struct nvm_tgt_dev *dev = pblk->dev; struct nvm_geo *geo = &dev->geo; @@ -528,11 +341,16 @@ static int pblk_recov_scan_oob(struct pblk *pblk, struct pblk_line *line, struct bio *bio; void *data; dma_addr_t dma_ppa_list, dma_meta_list; - u64 paddr; + __le64 *lba_list; + u64 paddr = 0; + bool padded = false; int rq_ppas, rq_len; int i, j; - int ret = 0; - int left_ppas = pblk_calc_sec_in_line(pblk, line); + int ret; + u64 left_ppas = pblk_sec_in_open_line(pblk, line); + + if (pblk_line_wp_is_unbalanced(pblk, line)) + pblk_warn(pblk, "recovering unbalanced line (%d)\n", line->id); ppa_list = p.ppa_list; meta_list = p.meta_list; @@ -541,7 +359,7 @@ static int pblk_recov_scan_oob(struct pblk *pblk, struct pblk_line *line, dma_ppa_list = p.dma_ppa_list; dma_meta_list = p.dma_meta_list; - *done = 1; + lba_list = emeta_to_lbas(pblk, line->emeta->buf); next_rq: memset(rqd, 0, pblk_g_rq_size); @@ -567,15 +385,13 @@ next_rq: rqd->dma_meta_list = dma_meta_list; if (pblk_io_aligned(pblk, rq_ppas)) - rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_SEQUENTIAL); - else - rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM); + rqd->is_seq = 1; +retry_rq: for (i = 0; i < rqd->nr_ppas; ) { struct ppa_addr ppa; int pos; - paddr = pblk_alloc_page(pblk, line, pblk->min_write_pgs); ppa = addr_to_gen_ppa(pblk, paddr, line->id); pos = pblk_ppa_to_pos(geo, ppa); @@ -585,9 +401,9 @@ next_rq: pos = pblk_ppa_to_pos(geo, ppa); } - for (j = 0; j < pblk->min_write_pgs; j++, i++, paddr++) + for (j = 0; j < pblk->min_write_pgs; j++, i++) rqd->ppa_list[i] = - addr_to_gen_ppa(pblk, paddr, line->id); + addr_to_gen_ppa(pblk, paddr + j, line->id); } ret = pblk_submit_io_sync(pblk, rqd); @@ -599,31 +415,33 @@ next_rq: atomic_dec(&pblk->inflight_io); - /* Reached the end of the written line */ + /* If a read fails, do a best effort by padding the line and retrying */ if (rqd->error) { - int nr_error_bits, bit; + int pad_distance, ret; - bit = find_first_bit((void *)&rqd->ppa_status, rqd->nr_ppas); - nr_error_bits = rqd->nr_ppas - bit; - - /* Roll back failed sectors */ - line->cur_sec -= nr_error_bits; - line->left_msecs += nr_error_bits; - bitmap_clear(line->map_bitmap, line->cur_sec, nr_error_bits); + if (padded) { + pblk_log_read_err(pblk, rqd); + return -EINTR; + } - left_ppas = 0; - rqd->nr_ppas = bit; + pad_distance = pblk_pad_distance(pblk, line); + ret = pblk_recov_pad_line(pblk, line, pad_distance); + if (ret) + return ret; - if (rqd->error != NVM_RSP_ERR_EMPTYPAGE) - *done = 0; + padded = true; + goto retry_rq; } for (i = 0; i < rqd->nr_ppas; i++) { u64 lba = le64_to_cpu(meta_list[i].lba); + lba_list[paddr++] = cpu_to_le64(lba); + if (lba == ADDR_EMPTY || lba > pblk->rl.nr_secs) continue; + line->nr_valid_lbas++; pblk_update_map(pblk, lba, rqd->ppa_list[i]); } @@ -631,7 +449,11 @@ next_rq: if (left_ppas > 0) goto next_rq; - return ret; +#ifdef CONFIG_NVM_PBLK_DEBUG + WARN_ON(padded && !pblk_line_is_full(line)); +#endif + + return 0; } /* Scan line for lbas on out of bound area */ @@ -645,7 +467,7 @@ static int pblk_recov_l2p_from_oob(struct pblk *pblk, struct pblk_line *line) struct pblk_recov_alloc p; void *data; dma_addr_t dma_ppa_list, dma_meta_list; - int done, ret = 0; + int ret = 0; meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, &dma_meta_list); if (!meta_list) @@ -660,7 +482,8 @@ static int pblk_recov_l2p_from_oob(struct pblk *pblk, struct pblk_line *line) goto free_meta_list; } - rqd = pblk_alloc_rqd(pblk, PBLK_READ); + rqd = mempool_alloc(&pblk->r_rq_pool, GFP_KERNEL); + memset(rqd, 0, pblk_g_rq_size); p.ppa_list = ppa_list; p.meta_list = meta_list; @@ -669,24 +492,17 @@ static int pblk_recov_l2p_from_oob(struct pblk *pblk, struct pblk_line *line) p.dma_ppa_list = dma_ppa_list; p.dma_meta_list = dma_meta_list; - ret = pblk_recov_scan_oob(pblk, line, p, &done); + ret = pblk_recov_scan_oob(pblk, line, p); if (ret) { - pblk_err(pblk, "could not recover L2P from OOB\n"); + pblk_err(pblk, "could not recover L2P form OOB\n"); goto out; } - if (!done) { - ret = pblk_recov_scan_all_oob(pblk, line, p); - if (ret) { - pblk_err(pblk, "could not recover L2P from OOB\n"); - goto out; - } - } - if (pblk_line_is_full(line)) pblk_line_recov_close(pblk, line); out: + mempool_free(rqd, &pblk->r_rq_pool); kfree(data); free_meta_list: nvm_dev_dma_free(dev->parent, meta_list, dma_meta_list); @@ -775,7 +591,7 @@ static void pblk_recov_wa_counters(struct pblk *pblk, } static int pblk_line_was_written(struct pblk_line *line, - struct pblk *pblk) + struct pblk *pblk) { struct pblk_line_meta *lm = &pblk->lm; @@ -801,6 +617,18 @@ static int pblk_line_was_written(struct pblk_line *line, return 1; } +static bool pblk_line_is_open(struct pblk *pblk, struct pblk_line *line) +{ + struct pblk_line_meta *lm = &pblk->lm; + int i; + + for (i = 0; i < lm->blk_per_line; i++) + if (line->chks[i].state & NVM_CHK_ST_OPEN) + return true; + + return false; +} + struct pblk_line *pblk_recov_l2p(struct pblk *pblk) { struct pblk_line_meta *lm = &pblk->lm; @@ -841,7 +669,7 @@ struct pblk_line *pblk_recov_l2p(struct pblk *pblk) continue; /* Lines that cannot be read are assumed as not written here */ - if (pblk_line_read_smeta(pblk, line)) + if (pblk_line_smeta_read(pblk, line)) continue; crc = pblk_calc_smeta_crc(pblk, smeta_buf); @@ -911,7 +739,12 @@ struct pblk_line *pblk_recov_l2p(struct pblk *pblk) line->emeta = emeta; memset(line->emeta->buf, 0, lm->emeta_len[0]); - if (pblk_line_read_emeta(pblk, line, line->emeta->buf)) { + if (pblk_line_is_open(pblk, line)) { + pblk_recov_l2p_from_oob(pblk, line); + goto next; + } + + if (pblk_line_emeta_read(pblk, line, line->emeta->buf)) { pblk_recov_l2p_from_oob(pblk, line); goto next; } @@ -935,6 +768,8 @@ next: spin_lock(&line->lock); line->state = PBLK_LINESTATE_CLOSED; + trace_pblk_line_state(pblk_disk_name(pblk), line->id, + line->state); move_list = pblk_line_gc_list(pblk, line); spin_unlock(&line->lock); @@ -942,26 +777,36 @@ next: list_move_tail(&line->list, move_list); spin_unlock(&l_mg->gc_lock); - kfree(line->map_bitmap); + mempool_free(line->map_bitmap, l_mg->bitmap_pool); line->map_bitmap = NULL; line->smeta = NULL; line->emeta = NULL; } else { - if (open_lines > 1) - pblk_err(pblk, "failed to recover L2P\n"); + spin_lock(&line->lock); + line->state = PBLK_LINESTATE_OPEN; + spin_unlock(&line->lock); + + line->emeta->mem = 0; + atomic_set(&line->emeta->sync, 0); + + trace_pblk_line_state(pblk_disk_name(pblk), line->id, + line->state); - open_lines++; - line->meta_line = meta_line; data_line = line; + line->meta_line = meta_line; + + open_lines++; } } - spin_lock(&l_mg->free_lock); if (!open_lines) { + spin_lock(&l_mg->free_lock); WARN_ON_ONCE(!test_and_clear_bit(meta_line, &l_mg->meta_bitmap)); + spin_unlock(&l_mg->free_lock); pblk_line_replace_data(pblk); } else { + spin_lock(&l_mg->free_lock); /* Allocate next line for preparation */ l_mg->data_next = pblk_line_get(pblk); if (l_mg->data_next) { @@ -969,8 +814,8 @@ next: l_mg->data_next->type = PBLK_LINETYPE_DATA; is_next = 1; } + spin_unlock(&l_mg->free_lock); } - spin_unlock(&l_mg->free_lock); if (is_next) pblk_line_erase(pblk, l_mg->data_next); @@ -998,7 +843,7 @@ int pblk_recov_pad(struct pblk *pblk) left_msecs = line->left_msecs; spin_unlock(&l_mg->free_lock); - ret = pblk_recov_pad_oob(pblk, line, left_msecs); + ret = pblk_recov_pad_line(pblk, line, left_msecs); if (ret) { pblk_err(pblk, "tear down padding failed (%d)\n", ret); return ret; diff --git a/drivers/lightnvm/pblk-rl.c b/drivers/lightnvm/pblk-rl.c index 6a0616a6fcaf..db55a1c89997 100644 --- a/drivers/lightnvm/pblk-rl.c +++ b/drivers/lightnvm/pblk-rl.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2016 CNEX Labs * Initial release: Javier Gonzalez <javier@cnexlabs.com> @@ -127,7 +128,7 @@ static void __pblk_rl_update_rates(struct pblk_rl *rl, } else if (free_blocks < rl->high) { int shift = rl->high_pw - rl->rb_windows_pw; int user_windows = free_blocks >> shift; - int user_max = user_windows << PBLK_MAX_REQ_ADDRS_PW; + int user_max = user_windows << ilog2(NVM_MAX_VLBA); rl->rb_user_max = user_max; rl->rb_gc_max = max - user_max; @@ -228,7 +229,7 @@ void pblk_rl_init(struct pblk_rl *rl, int budget) rl->rsv_blocks = min_blocks; /* This will always be a power-of-2 */ - rb_windows = budget / PBLK_MAX_REQ_ADDRS; + rb_windows = budget / NVM_MAX_VLBA; rl->rb_windows_pw = get_count_order(rb_windows); /* To start with, all buffer is available to user I/O writers */ diff --git a/drivers/lightnvm/pblk-sysfs.c b/drivers/lightnvm/pblk-sysfs.c index 9fc3dfa168b4..2d2818155aa8 100644 --- a/drivers/lightnvm/pblk-sysfs.c +++ b/drivers/lightnvm/pblk-sysfs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2016 CNEX Labs * Initial release: Javier Gonzalez <javier@cnexlabs.com> @@ -262,8 +263,14 @@ static ssize_t pblk_sysfs_lines(struct pblk *pblk, char *page) sec_in_line = l_mg->data_line->sec_in_line; meta_weight = bitmap_weight(&l_mg->meta_bitmap, PBLK_DATA_LINES); - map_weight = bitmap_weight(l_mg->data_line->map_bitmap, + + spin_lock(&l_mg->data_line->lock); + if (l_mg->data_line->map_bitmap) + map_weight = bitmap_weight(l_mg->data_line->map_bitmap, lm->sec_per_line); + else + map_weight = 0; + spin_unlock(&l_mg->data_line->lock); } spin_unlock(&l_mg->free_lock); @@ -337,7 +344,6 @@ static ssize_t pblk_get_write_amp(u64 user, u64 gc, u64 pad, { int sz; - sz = snprintf(page, PAGE_SIZE, "user:%lld gc:%lld pad:%lld WA:", user, gc, pad); @@ -349,7 +355,7 @@ static ssize_t pblk_get_write_amp(u64 user, u64 gc, u64 pad, u32 wa_frac; wa_int = (user + gc + pad) * 100000; - wa_int = div_u64(wa_int, user); + wa_int = div64_u64(wa_int, user); wa_int = div_u64_rem(wa_int, 100000, &wa_frac); sz += snprintf(page + sz, PAGE_SIZE - sz, "%llu.%05u\n", diff --git a/drivers/lightnvm/pblk-trace.h b/drivers/lightnvm/pblk-trace.h new file mode 100644 index 000000000000..679e5c458ca6 --- /dev/null +++ b/drivers/lightnvm/pblk-trace.h @@ -0,0 +1,145 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM pblk + +#if !defined(_TRACE_PBLK_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_PBLK_H + +#include <linux/tracepoint.h> + +struct ppa_addr; + +#define show_chunk_flags(state) __print_flags(state, "", \ + { NVM_CHK_ST_FREE, "FREE", }, \ + { NVM_CHK_ST_CLOSED, "CLOSED", }, \ + { NVM_CHK_ST_OPEN, "OPEN", }, \ + { NVM_CHK_ST_OFFLINE, "OFFLINE", }) + +#define show_line_state(state) __print_symbolic(state, \ + { PBLK_LINESTATE_NEW, "NEW", }, \ + { PBLK_LINESTATE_FREE, "FREE", }, \ + { PBLK_LINESTATE_OPEN, "OPEN", }, \ + { PBLK_LINESTATE_CLOSED, "CLOSED", }, \ + { PBLK_LINESTATE_GC, "GC", }, \ + { PBLK_LINESTATE_BAD, "BAD", }, \ + { PBLK_LINESTATE_CORRUPT, "CORRUPT" }) + + +#define show_pblk_state(state) __print_symbolic(state, \ + { PBLK_STATE_RUNNING, "RUNNING", }, \ + { PBLK_STATE_STOPPING, "STOPPING", }, \ + { PBLK_STATE_RECOVERING, "RECOVERING", }, \ + { PBLK_STATE_STOPPED, "STOPPED" }) + +#define show_chunk_erase_state(state) __print_symbolic(state, \ + { PBLK_CHUNK_RESET_START, "START", }, \ + { PBLK_CHUNK_RESET_DONE, "OK", }, \ + { PBLK_CHUNK_RESET_FAILED, "FAILED" }) + + +TRACE_EVENT(pblk_chunk_reset, + + TP_PROTO(const char *name, struct ppa_addr *ppa, int state), + + TP_ARGS(name, ppa, state), + + TP_STRUCT__entry( + __string(name, name) + __field(u64, ppa) + __field(int, state); + ), + + TP_fast_assign( + __assign_str(name, name); + __entry->ppa = ppa->ppa; + __entry->state = state; + ), + + TP_printk("dev=%s grp=%llu pu=%llu chk=%llu state=%s", __get_str(name), + (u64)(((struct ppa_addr *)(&__entry->ppa))->m.grp), + (u64)(((struct ppa_addr *)(&__entry->ppa))->m.pu), + (u64)(((struct ppa_addr *)(&__entry->ppa))->m.chk), + show_chunk_erase_state((int)__entry->state)) + +); + +TRACE_EVENT(pblk_chunk_state, + + TP_PROTO(const char *name, struct ppa_addr *ppa, int state), + + TP_ARGS(name, ppa, state), + + TP_STRUCT__entry( + __string(name, name) + __field(u64, ppa) + __field(int, state); + ), + + TP_fast_assign( + __assign_str(name, name); + __entry->ppa = ppa->ppa; + __entry->state = state; + ), + + TP_printk("dev=%s grp=%llu pu=%llu chk=%llu state=%s", __get_str(name), + (u64)(((struct ppa_addr *)(&__entry->ppa))->m.grp), + (u64)(((struct ppa_addr *)(&__entry->ppa))->m.pu), + (u64)(((struct ppa_addr *)(&__entry->ppa))->m.chk), + show_chunk_flags((int)__entry->state)) + +); + +TRACE_EVENT(pblk_line_state, + + TP_PROTO(const char *name, int line, int state), + + TP_ARGS(name, line, state), + + TP_STRUCT__entry( + __string(name, name) + __field(int, line) + __field(int, state); + ), + + TP_fast_assign( + __assign_str(name, name); + __entry->line = line; + __entry->state = state; + ), + + TP_printk("dev=%s line=%d state=%s", __get_str(name), + (int)__entry->line, + show_line_state((int)__entry->state)) + +); + +TRACE_EVENT(pblk_state, + + TP_PROTO(const char *name, int state), + + TP_ARGS(name, state), + + TP_STRUCT__entry( + __string(name, name) + __field(int, state); + ), + + TP_fast_assign( + __assign_str(name, name); + __entry->state = state; + ), + + TP_printk("dev=%s state=%s", __get_str(name), + show_pblk_state((int)__entry->state)) + +); + +#endif /* !defined(_TRACE_PBLK_H) || defined(TRACE_HEADER_MULTI_READ) */ + +/* This part must be outside protection */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../../drivers/lightnvm +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE pblk-trace +#include <trace/define_trace.h> diff --git a/drivers/lightnvm/pblk-write.c b/drivers/lightnvm/pblk-write.c index ee774a86cf1e..fa8726493b39 100644 --- a/drivers/lightnvm/pblk-write.c +++ b/drivers/lightnvm/pblk-write.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2016 CNEX Labs * Initial release: Javier Gonzalez <javier@cnexlabs.com> @@ -16,6 +17,7 @@ */ #include "pblk.h" +#include "pblk-trace.h" static unsigned long pblk_end_w_bio(struct pblk *pblk, struct nvm_rq *rqd, struct pblk_c_ctx *c_ctx) @@ -81,8 +83,7 @@ static void pblk_complete_write(struct pblk *pblk, struct nvm_rq *rqd, #ifdef CONFIG_NVM_PBLK_DEBUG atomic_long_sub(c_ctx->nr_valid, &pblk->inflight_writes); #endif - - pblk_up_rq(pblk, rqd->ppa_list, rqd->nr_ppas, c_ctx->lun_bitmap); + pblk_up_rq(pblk, c_ctx->lun_bitmap); pos = pblk_rb_sync_init(&pblk->rwb, &flags); if (pos == c_ctx->sentry) { @@ -106,14 +107,12 @@ retry: /* Map remaining sectors in chunk, starting from ppa */ static void pblk_map_remaining(struct pblk *pblk, struct ppa_addr *ppa) { - struct nvm_tgt_dev *dev = pblk->dev; - struct nvm_geo *geo = &dev->geo; struct pblk_line *line; struct ppa_addr map_ppa = *ppa; u64 paddr; int done = 0; - line = &pblk->lines[pblk_ppa_to_line(*ppa)]; + line = pblk_ppa_to_line(pblk, *ppa); spin_lock(&line->lock); while (!done) { @@ -125,15 +124,7 @@ static void pblk_map_remaining(struct pblk *pblk, struct ppa_addr *ppa) if (!test_and_set_bit(paddr, line->invalid_bitmap)) le32_add_cpu(line->vsc, -1); - if (geo->version == NVM_OCSSD_SPEC_12) { - map_ppa.ppa++; - if (map_ppa.g.pg == geo->num_pg) - done = 1; - } else { - map_ppa.m.sec++; - if (map_ppa.m.sec == geo->clba) - done = 1; - } + done = nvm_next_ppa_in_chk(pblk->dev, &map_ppa); } line->w_err_gc->has_write_err = 1; @@ -149,12 +140,11 @@ static void pblk_prepare_resubmit(struct pblk *pblk, unsigned int sentry, struct pblk_w_ctx *w_ctx; struct ppa_addr ppa_l2p; int flags; - unsigned int pos, i; + unsigned int i; spin_lock(&pblk->trans_lock); - pos = sentry; for (i = 0; i < nr_entries; i++) { - entry = &rb->entries[pos]; + entry = &rb->entries[pblk_rb_ptr_wrap(rb, sentry, i)]; w_ctx = &entry->w_ctx; /* Check if the lba has been overwritten */ @@ -168,13 +158,11 @@ static void pblk_prepare_resubmit(struct pblk *pblk, unsigned int sentry, /* Release flags on write context. Protect from writes */ smp_store_release(&w_ctx->flags, flags); - /* Decrese the reference count to the line as we will + /* Decrease the reference count to the line as we will * re-map these entries */ - line = &pblk->lines[pblk_ppa_to_line(w_ctx->ppa)]; + line = pblk_ppa_to_line(pblk, w_ctx->ppa); kref_put(&line->ref, pblk_line_put); - - pos = (pos + 1) & (rb->nr_entries - 1); } spin_unlock(&pblk->trans_lock); } @@ -208,19 +196,14 @@ static void pblk_submit_rec(struct work_struct *work) struct pblk *pblk = recovery->pblk; struct nvm_rq *rqd = recovery->rqd; struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd); - struct ppa_addr *ppa_list; + struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); pblk_log_write_err(pblk, rqd); - if (rqd->nr_ppas == 1) - ppa_list = &rqd->ppa_addr; - else - ppa_list = rqd->ppa_list; - pblk_map_remaining(pblk, ppa_list); pblk_queue_resubmit(pblk, c_ctx); - pblk_up_rq(pblk, rqd->ppa_list, rqd->nr_ppas, c_ctx->lun_bitmap); + pblk_up_rq(pblk, c_ctx->lun_bitmap); if (c_ctx->nr_padded) pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid, c_ctx->nr_padded); @@ -257,11 +240,13 @@ static void pblk_end_io_write(struct nvm_rq *rqd) if (rqd->error) { pblk_end_w_fail(pblk, rqd); return; - } + } else { + if (trace_pblk_chunk_state_enabled()) + pblk_check_chunk_state_update(pblk, rqd); #ifdef CONFIG_NVM_PBLK_DEBUG - else WARN_ONCE(rqd->bio->bi_status, "pblk: corrupted write error\n"); #endif + } pblk_complete_write(pblk, rqd, c_ctx); atomic_dec(&pblk->inflight_io); @@ -273,14 +258,18 @@ static void pblk_end_io_write_meta(struct nvm_rq *rqd) struct pblk_g_ctx *m_ctx = nvm_rq_to_pdu(rqd); struct pblk_line *line = m_ctx->private; struct pblk_emeta *emeta = line->emeta; + struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); int sync; - pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas); + pblk_up_chunk(pblk, ppa_list[0]); if (rqd->error) { pblk_log_write_err(pblk, rqd); pblk_err(pblk, "metadata I/O failed. Line %d\n", line->id); line->w_err_gc->has_write_err = 1; + } else { + if (trace_pblk_chunk_state_enabled()) + pblk_check_chunk_state_update(pblk, rqd); } sync = atomic_add_return(rqd->nr_ppas, &emeta->sync); @@ -294,27 +283,16 @@ static void pblk_end_io_write_meta(struct nvm_rq *rqd) } static int pblk_alloc_w_rq(struct pblk *pblk, struct nvm_rq *rqd, - unsigned int nr_secs, - nvm_end_io_fn(*end_io)) + unsigned int nr_secs, nvm_end_io_fn(*end_io)) { - struct nvm_tgt_dev *dev = pblk->dev; - /* Setup write request */ rqd->opcode = NVM_OP_PWRITE; rqd->nr_ppas = nr_secs; - rqd->flags = pblk_set_progr_mode(pblk, PBLK_WRITE); + rqd->is_seq = 1; rqd->private = pblk; rqd->end_io = end_io; - rqd->meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, - &rqd->dma_meta_list); - if (!rqd->meta_list) - return -ENOMEM; - - rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size; - rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size; - - return 0; + return pblk_alloc_rqd_meta(pblk, rqd); } static int pblk_setup_w_rq(struct pblk *pblk, struct nvm_rq *rqd, @@ -375,6 +353,7 @@ int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line) struct pblk_line_mgmt *l_mg = &pblk->l_mg; struct pblk_line_meta *lm = &pblk->lm; struct pblk_emeta *emeta = meta_line->emeta; + struct ppa_addr *ppa_list; struct pblk_g_ctx *m_ctx; struct bio *bio; struct nvm_rq *rqd; @@ -409,22 +388,22 @@ int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line) if (ret) goto fail_free_bio; + ppa_list = nvm_rq_to_ppa_list(rqd); for (i = 0; i < rqd->nr_ppas; ) { spin_lock(&meta_line->lock); paddr = __pblk_alloc_page(pblk, meta_line, rq_ppas); spin_unlock(&meta_line->lock); for (j = 0; j < rq_ppas; j++, i++, paddr++) - rqd->ppa_list[i] = addr_to_gen_ppa(pblk, paddr, id); + ppa_list[i] = addr_to_gen_ppa(pblk, paddr, id); } + spin_lock(&l_mg->close_lock); emeta->mem += rq_len; - if (emeta->mem >= lm->emeta_len[0]) { - spin_lock(&l_mg->close_lock); + if (emeta->mem >= lm->emeta_len[0]) list_del(&meta_line->list); - spin_unlock(&l_mg->close_lock); - } + spin_unlock(&l_mg->close_lock); - pblk_down_page(pblk, rqd->ppa_list, rqd->nr_ppas); + pblk_down_chunk(pblk, ppa_list[0]); ret = pblk_submit_io(pblk, rqd); if (ret) { @@ -435,7 +414,7 @@ int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line) return NVM_IO_OK; fail_rollback: - pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas); + pblk_up_chunk(pblk, ppa_list[0]); spin_lock(&l_mg->close_lock); pblk_dealloc_page(pblk, meta_line, rq_ppas); list_add(&meta_line->list, &meta_line->list); @@ -491,14 +470,15 @@ static struct pblk_line *pblk_should_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line; spin_lock(&l_mg->close_lock); -retry: if (list_empty(&l_mg->emeta_list)) { spin_unlock(&l_mg->close_lock); return NULL; } meta_line = list_first_entry(&l_mg->emeta_list, struct pblk_line, list); - if (meta_line->emeta->mem >= lm->emeta_len[0]) - goto retry; + if (meta_line->emeta->mem >= lm->emeta_len[0]) { + spin_unlock(&l_mg->close_lock); + return NULL; + } spin_unlock(&l_mg->close_lock); if (!pblk_valid_meta_ppa(pblk, meta_line, data_rqd)) diff --git a/drivers/lightnvm/pblk.h b/drivers/lightnvm/pblk.h index 4760af7b6499..02bb2e98f8a9 100644 --- a/drivers/lightnvm/pblk.h +++ b/drivers/lightnvm/pblk.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2015 IT University of Copenhagen (rrpc.h) * Copyright (C) 2016 CNEX Labs @@ -37,8 +38,6 @@ #define PBLK_SECTOR (512) #define PBLK_EXPOSED_PAGE_SIZE (4096) -#define PBLK_MAX_REQ_ADDRS (64) -#define PBLK_MAX_REQ_ADDRS_PW (6) #define PBLK_NR_CLOSE_JOBS (4) @@ -81,6 +80,12 @@ enum { PBLK_BLK_ST_CLOSED = 0x2, }; +enum { + PBLK_CHUNK_RESET_START, + PBLK_CHUNK_RESET_DONE, + PBLK_CHUNK_RESET_FAILED, +}; + struct pblk_sec_meta { u64 reserved; __le64 lba; @@ -99,8 +104,8 @@ enum { PBLK_RL_LOW = 4 }; -#define pblk_dma_meta_size (sizeof(struct pblk_sec_meta) * PBLK_MAX_REQ_ADDRS) -#define pblk_dma_ppa_size (sizeof(u64) * PBLK_MAX_REQ_ADDRS) +#define pblk_dma_meta_size (sizeof(struct pblk_sec_meta) * NVM_MAX_VLBA) +#define pblk_dma_ppa_size (sizeof(u64) * NVM_MAX_VLBA) /* write buffer completion context */ struct pblk_c_ctx { @@ -198,6 +203,11 @@ struct pblk_rb { * will be 4KB */ + unsigned int back_thres; /* Threshold that shall be maintained by + * the backpointer in order to respect + * geo->mw_cunits on a per chunk basis + */ + struct list_head pages; /* List of data pages */ spinlock_t w_lock; /* Write lock */ @@ -218,8 +228,8 @@ struct pblk_lun { struct pblk_gc_rq { struct pblk_line *line; void *data; - u64 paddr_list[PBLK_MAX_REQ_ADDRS]; - u64 lba_list[PBLK_MAX_REQ_ADDRS]; + u64 paddr_list[NVM_MAX_VLBA]; + u64 lba_list[NVM_MAX_VLBA]; int nr_secs; int secs_to_gc; struct list_head list; @@ -532,6 +542,10 @@ struct pblk_line_mgmt { struct pblk_emeta *eline_meta[PBLK_DATA_LINES]; unsigned long meta_bitmap; + /* Cache and mempool for map/invalid bitmaps */ + struct kmem_cache *bitmap_cache; + mempool_t *bitmap_pool; + /* Helpers for fast bitmap calculations */ unsigned long *bb_template; unsigned long *bb_aux; @@ -725,10 +739,8 @@ struct pblk_line_ws { /* * pblk ring buffer operations */ -int pblk_rb_init(struct pblk_rb *rb, struct pblk_rb_entry *rb_entry_base, - unsigned int power_size, unsigned int power_seg_sz); -unsigned int pblk_rb_calculate_size(unsigned int nr_entries); -void *pblk_rb_entries_ref(struct pblk_rb *rb); +int pblk_rb_init(struct pblk_rb *rb, unsigned int size, unsigned int threshold, + unsigned int seg_sz); int pblk_rb_may_write_user(struct pblk_rb *rb, struct bio *bio, unsigned int nr_entries, unsigned int *pos); int pblk_rb_may_write_gc(struct pblk_rb *rb, unsigned int nr_entries, @@ -751,8 +763,8 @@ unsigned int pblk_rb_read_commit(struct pblk_rb *rb, unsigned int entries); unsigned int pblk_rb_sync_init(struct pblk_rb *rb, unsigned long *flags); unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries); -struct pblk_rb_entry *pblk_rb_sync_scan_entry(struct pblk_rb *rb, - struct ppa_addr *ppa); +unsigned int pblk_rb_ptr_wrap(struct pblk_rb *rb, unsigned int p, + unsigned int nr_entries); void pblk_rb_sync_end(struct pblk_rb *rb, unsigned long *flags); unsigned int pblk_rb_flush_point_count(struct pblk_rb *rb); @@ -762,7 +774,7 @@ unsigned int pblk_rb_wrap_pos(struct pblk_rb *rb, unsigned int pos); int pblk_rb_tear_down_check(struct pblk_rb *rb); int pblk_rb_pos_oob(struct pblk_rb *rb, u64 pos); -void pblk_rb_data_free(struct pblk_rb *rb); +void pblk_rb_free(struct pblk_rb *rb); ssize_t pblk_rb_sysfs(struct pblk_rb *rb, char *buf); /* @@ -770,11 +782,13 @@ ssize_t pblk_rb_sysfs(struct pblk_rb *rb, char *buf); */ struct nvm_rq *pblk_alloc_rqd(struct pblk *pblk, int type); void pblk_free_rqd(struct pblk *pblk, struct nvm_rq *rqd, int type); +int pblk_alloc_rqd_meta(struct pblk *pblk, struct nvm_rq *rqd); +void pblk_free_rqd_meta(struct pblk *pblk, struct nvm_rq *rqd); void pblk_set_sec_per_write(struct pblk *pblk, int sec_per_write); int pblk_setup_w_rec_rq(struct pblk *pblk, struct nvm_rq *rqd, struct pblk_c_ctx *c_ctx); void pblk_discard(struct pblk *pblk, struct bio *bio); -struct nvm_chk_meta *pblk_chunk_get_info(struct pblk *pblk); +struct nvm_chk_meta *pblk_get_chunk_meta(struct pblk *pblk); struct nvm_chk_meta *pblk_chunk_get_off(struct pblk *pblk, struct nvm_chk_meta *lp, struct ppa_addr ppa); @@ -782,13 +796,17 @@ void pblk_log_write_err(struct pblk *pblk, struct nvm_rq *rqd); void pblk_log_read_err(struct pblk *pblk, struct nvm_rq *rqd); int pblk_submit_io(struct pblk *pblk, struct nvm_rq *rqd); int pblk_submit_io_sync(struct pblk *pblk, struct nvm_rq *rqd); +int pblk_submit_io_sync_sem(struct pblk *pblk, struct nvm_rq *rqd); int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line); +void pblk_check_chunk_state_update(struct pblk *pblk, struct nvm_rq *rqd); struct bio *pblk_bio_map_addr(struct pblk *pblk, void *data, unsigned int nr_secs, unsigned int len, int alloc_type, gfp_t gfp_mask); struct pblk_line *pblk_line_get(struct pblk *pblk); struct pblk_line *pblk_line_get_first_data(struct pblk *pblk); struct pblk_line *pblk_line_replace_data(struct pblk *pblk); +void pblk_ppa_to_line_put(struct pblk *pblk, struct ppa_addr ppa); +void pblk_rq_to_line_put(struct pblk *pblk, struct nvm_rq *rqd); int pblk_line_recov_alloc(struct pblk *pblk, struct pblk_line *line); void pblk_line_recov_close(struct pblk *pblk, struct pblk_line *line); struct pblk_line *pblk_line_get_data(struct pblk *pblk); @@ -806,8 +824,8 @@ void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv, void (*work)(struct work_struct *), gfp_t gfp_mask, struct workqueue_struct *wq); u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line); -int pblk_line_read_smeta(struct pblk *pblk, struct pblk_line *line); -int pblk_line_read_emeta(struct pblk *pblk, struct pblk_line *line, +int pblk_line_smeta_read(struct pblk *pblk, struct pblk_line *line); +int pblk_line_emeta_read(struct pblk *pblk, struct pblk_line *line, void *emeta_buf); int pblk_blk_erase_async(struct pblk *pblk, struct ppa_addr erase_ppa); void pblk_line_put(struct kref *ref); @@ -819,12 +837,11 @@ u64 pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs); u64 __pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs); int pblk_calc_secs(struct pblk *pblk, unsigned long secs_avail, unsigned long secs_to_flush); -void pblk_up_page(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas); -void pblk_down_rq(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas, +void pblk_down_rq(struct pblk *pblk, struct ppa_addr ppa, unsigned long *lun_bitmap); -void pblk_down_page(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas); -void pblk_up_rq(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas, - unsigned long *lun_bitmap); +void pblk_down_chunk(struct pblk *pblk, struct ppa_addr ppa); +void pblk_up_chunk(struct pblk *pblk, struct ppa_addr ppa); +void pblk_up_rq(struct pblk *pblk, unsigned long *lun_bitmap); int pblk_bio_add_pages(struct pblk *pblk, struct bio *bio, gfp_t flags, int nr_pages); void pblk_bio_free_pages(struct pblk *pblk, struct bio *bio, int off, @@ -976,17 +993,15 @@ static inline int pblk_line_vsc(struct pblk_line *line) return le32_to_cpu(*line->vsc); } -static inline int pblk_pad_distance(struct pblk *pblk) +static inline int pblk_ppa_to_line_id(struct ppa_addr p) { - struct nvm_tgt_dev *dev = pblk->dev; - struct nvm_geo *geo = &dev->geo; - - return geo->mw_cunits * geo->all_luns * geo->ws_opt; + return p.a.blk; } -static inline int pblk_ppa_to_line(struct ppa_addr p) +static inline struct pblk_line *pblk_ppa_to_line(struct pblk *pblk, + struct ppa_addr p) { - return p.a.blk; + return &pblk->lines[pblk_ppa_to_line_id(p)]; } static inline int pblk_ppa_to_pos(struct nvm_geo *geo, struct ppa_addr p) @@ -1034,6 +1049,25 @@ static inline struct ppa_addr addr_to_gen_ppa(struct pblk *pblk, u64 paddr, return ppa; } +static inline struct nvm_chk_meta *pblk_dev_ppa_to_chunk(struct pblk *pblk, + struct ppa_addr p) +{ + struct nvm_tgt_dev *dev = pblk->dev; + struct nvm_geo *geo = &dev->geo; + struct pblk_line *line = pblk_ppa_to_line(pblk, p); + int pos = pblk_ppa_to_pos(geo, p); + + return &line->chks[pos]; +} + +static inline u64 pblk_dev_ppa_to_chunk_addr(struct pblk *pblk, + struct ppa_addr p) +{ + struct nvm_tgt_dev *dev = pblk->dev; + + return dev_to_chunk_addr(dev->parent, &pblk->addrf, p); +} + static inline u64 pblk_dev_ppa_to_line_addr(struct pblk *pblk, struct ppa_addr p) { @@ -1067,86 +1101,16 @@ static inline u64 pblk_dev_ppa_to_line_addr(struct pblk *pblk, static inline struct ppa_addr pblk_ppa32_to_ppa64(struct pblk *pblk, u32 ppa32) { - struct ppa_addr ppa64; - - ppa64.ppa = 0; - - if (ppa32 == -1) { - ppa64.ppa = ADDR_EMPTY; - } else if (ppa32 & (1U << 31)) { - ppa64.c.line = ppa32 & ((~0U) >> 1); - ppa64.c.is_cached = 1; - } else { - struct nvm_tgt_dev *dev = pblk->dev; - struct nvm_geo *geo = &dev->geo; - - if (geo->version == NVM_OCSSD_SPEC_12) { - struct nvm_addrf_12 *ppaf = - (struct nvm_addrf_12 *)&pblk->addrf; - - ppa64.g.ch = (ppa32 & ppaf->ch_mask) >> - ppaf->ch_offset; - ppa64.g.lun = (ppa32 & ppaf->lun_mask) >> - ppaf->lun_offset; - ppa64.g.blk = (ppa32 & ppaf->blk_mask) >> - ppaf->blk_offset; - ppa64.g.pg = (ppa32 & ppaf->pg_mask) >> - ppaf->pg_offset; - ppa64.g.pl = (ppa32 & ppaf->pln_mask) >> - ppaf->pln_offset; - ppa64.g.sec = (ppa32 & ppaf->sec_mask) >> - ppaf->sec_offset; - } else { - struct nvm_addrf *lbaf = &pblk->addrf; - - ppa64.m.grp = (ppa32 & lbaf->ch_mask) >> - lbaf->ch_offset; - ppa64.m.pu = (ppa32 & lbaf->lun_mask) >> - lbaf->lun_offset; - ppa64.m.chk = (ppa32 & lbaf->chk_mask) >> - lbaf->chk_offset; - ppa64.m.sec = (ppa32 & lbaf->sec_mask) >> - lbaf->sec_offset; - } - } + struct nvm_tgt_dev *dev = pblk->dev; - return ppa64; + return nvm_ppa32_to_ppa64(dev->parent, &pblk->addrf, ppa32); } static inline u32 pblk_ppa64_to_ppa32(struct pblk *pblk, struct ppa_addr ppa64) { - u32 ppa32 = 0; - - if (ppa64.ppa == ADDR_EMPTY) { - ppa32 = ~0U; - } else if (ppa64.c.is_cached) { - ppa32 |= ppa64.c.line; - ppa32 |= 1U << 31; - } else { - struct nvm_tgt_dev *dev = pblk->dev; - struct nvm_geo *geo = &dev->geo; - - if (geo->version == NVM_OCSSD_SPEC_12) { - struct nvm_addrf_12 *ppaf = - (struct nvm_addrf_12 *)&pblk->addrf; - - ppa32 |= ppa64.g.ch << ppaf->ch_offset; - ppa32 |= ppa64.g.lun << ppaf->lun_offset; - ppa32 |= ppa64.g.blk << ppaf->blk_offset; - ppa32 |= ppa64.g.pg << ppaf->pg_offset; - ppa32 |= ppa64.g.pl << ppaf->pln_offset; - ppa32 |= ppa64.g.sec << ppaf->sec_offset; - } else { - struct nvm_addrf *lbaf = &pblk->addrf; - - ppa32 |= ppa64.m.grp << lbaf->ch_offset; - ppa32 |= ppa64.m.pu << lbaf->lun_offset; - ppa32 |= ppa64.m.chk << lbaf->chk_offset; - ppa32 |= ppa64.m.sec << lbaf->sec_offset; - } - } + struct nvm_tgt_dev *dev = pblk->dev; - return ppa32; + return nvm_ppa64_to_ppa32(dev->parent, &pblk->addrf, ppa64); } static inline struct ppa_addr pblk_trans_map_get(struct pblk *pblk, @@ -1255,44 +1219,6 @@ static inline u32 pblk_calc_emeta_crc(struct pblk *pblk, return crc; } -static inline int pblk_set_progr_mode(struct pblk *pblk, int type) -{ - struct nvm_tgt_dev *dev = pblk->dev; - struct nvm_geo *geo = &dev->geo; - int flags; - - if (geo->version == NVM_OCSSD_SPEC_20) - return 0; - - flags = geo->pln_mode >> 1; - - if (type == PBLK_WRITE) - flags |= NVM_IO_SCRAMBLE_ENABLE; - - return flags; -} - -enum { - PBLK_READ_RANDOM = 0, - PBLK_READ_SEQUENTIAL = 1, -}; - -static inline int pblk_set_read_mode(struct pblk *pblk, int type) -{ - struct nvm_tgt_dev *dev = pblk->dev; - struct nvm_geo *geo = &dev->geo; - int flags; - - if (geo->version == NVM_OCSSD_SPEC_20) - return 0; - - flags = NVM_IO_SUSPEND | NVM_IO_SCRAMBLE_ENABLE; - if (type == PBLK_READ_SEQUENTIAL) - flags |= geo->pln_mode >> 1; - - return flags; -} - static inline int pblk_io_aligned(struct pblk *pblk, int nr_secs) { return !(nr_secs % pblk->min_write_pgs); @@ -1375,9 +1301,7 @@ static inline int pblk_boundary_ppa_checks(struct nvm_tgt_dev *tgt_dev, static inline int pblk_check_io(struct pblk *pblk, struct nvm_rq *rqd) { struct nvm_tgt_dev *dev = pblk->dev; - struct ppa_addr *ppa_list; - - ppa_list = (rqd->nr_ppas > 1) ? rqd->ppa_list : &rqd->ppa_addr; + struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); if (pblk_boundary_ppa_checks(dev, ppa_list, rqd->nr_ppas)) { WARN_ON(1); @@ -1386,12 +1310,10 @@ static inline int pblk_check_io(struct pblk *pblk, struct nvm_rq *rqd) if (rqd->opcode == NVM_OP_PWRITE) { struct pblk_line *line; - struct ppa_addr ppa; int i; for (i = 0; i < rqd->nr_ppas; i++) { - ppa = ppa_list[i]; - line = &pblk->lines[pblk_ppa_to_line(ppa)]; + line = pblk_ppa_to_line(pblk, ppa_list[i]); spin_lock(&line->lock); if (line->state != PBLK_LINESTATE_OPEN) { @@ -1441,4 +1363,11 @@ static inline void pblk_setup_uuid(struct pblk *pblk) uuid_le_gen(&uuid); memcpy(pblk->instance_uuid, uuid.b, 16); } + +static inline char *pblk_disk_name(struct pblk *pblk) +{ + struct gendisk *disk = pblk->disk; + + return disk->disk_name; +} #endif /* PBLK_H_ */ diff --git a/drivers/md/bcache/alloc.c b/drivers/md/bcache/alloc.c index 7a28232d868b..5002838ea476 100644 --- a/drivers/md/bcache/alloc.c +++ b/drivers/md/bcache/alloc.c @@ -484,7 +484,7 @@ int __bch_bucket_alloc_set(struct cache_set *c, unsigned int reserve, int i; lockdep_assert_held(&c->bucket_lock); - BUG_ON(!n || n > c->caches_loaded || n > 8); + BUG_ON(!n || n > c->caches_loaded || n > MAX_CACHES_PER_SET); bkey_init(k); diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h index 954dad29e6e8..b61b83bbcfff 100644 --- a/drivers/md/bcache/bcache.h +++ b/drivers/md/bcache/bcache.h @@ -1004,7 +1004,7 @@ void bch_open_buckets_free(struct cache_set *c); int bch_cache_allocator_start(struct cache *ca); void bch_debug_exit(void); -void bch_debug_init(struct kobject *kobj); +void bch_debug_init(void); void bch_request_exit(void); int bch_request_init(void); diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c index e7d4817681f2..3f4211b5cd33 100644 --- a/drivers/md/bcache/btree.c +++ b/drivers/md/bcache/btree.c @@ -2434,7 +2434,7 @@ static int refill_keybuf_fn(struct btree_op *op, struct btree *b, struct keybuf *buf = refill->buf; int ret = MAP_CONTINUE; - if (bkey_cmp(k, refill->end) >= 0) { + if (bkey_cmp(k, refill->end) > 0) { ret = MAP_DONE; goto out; } diff --git a/drivers/md/bcache/closure.h b/drivers/md/bcache/closure.h index eca0d496b686..c88cdc4ae4ec 100644 --- a/drivers/md/bcache/closure.h +++ b/drivers/md/bcache/closure.h @@ -345,7 +345,8 @@ do { \ } while (0) /** - * closure_return - finish execution of a closure, with destructor + * closure_return_with_destructor - finish execution of a closure, + * with destructor * * Works like closure_return(), except @destructor will be called when all * outstanding refs on @cl have been dropped; @destructor may be used to safely diff --git a/drivers/md/bcache/debug.c b/drivers/md/bcache/debug.c index 06da66b2488a..8f448b9c96a1 100644 --- a/drivers/md/bcache/debug.c +++ b/drivers/md/bcache/debug.c @@ -253,7 +253,7 @@ void bch_debug_exit(void) debugfs_remove_recursive(bcache_debug); } -void __init bch_debug_init(struct kobject *kobj) +void __init bch_debug_init(void) { /* * it is unnecessary to check return value of diff --git a/drivers/md/bcache/extents.c b/drivers/md/bcache/extents.c index c809724e6571..956004366699 100644 --- a/drivers/md/bcache/extents.c +++ b/drivers/md/bcache/extents.c @@ -553,7 +553,7 @@ static bool bch_extent_bad(struct btree_keys *bk, const struct bkey *k) for (i = 0; i < KEY_PTRS(k); i++) { stale = ptr_stale(b->c, k, i); - btree_bug_on(stale > 96, b, + btree_bug_on(stale > BUCKET_GC_GEN_MAX, b, "key too stale: %i, need_gc %u", stale, b->c->need_gc); diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index 51be355a3309..3bf35914bb57 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c @@ -395,7 +395,7 @@ static bool check_should_bypass(struct cached_dev *dc, struct bio *bio) * unless the read-ahead request is for metadata (eg, for gfs2). */ if (bio->bi_opf & (REQ_RAHEAD|REQ_BACKGROUND) && - !(bio->bi_opf & REQ_META)) + !(bio->bi_opf & REQ_PRIO)) goto skip; if (bio->bi_iter.bi_sector & (c->sb.block_size - 1) || @@ -850,7 +850,7 @@ static void cached_dev_read_done_bh(struct closure *cl) bch_mark_cache_accounting(s->iop.c, s->d, !s->cache_missed, s->iop.bypass); - trace_bcache_read(s->orig_bio, !s->cache_miss, s->iop.bypass); + trace_bcache_read(s->orig_bio, !s->cache_missed, s->iop.bypass); if (s->iop.status) continue_at_nobarrier(cl, cached_dev_read_error, bcache_wq); @@ -877,7 +877,7 @@ static int cached_dev_cache_miss(struct btree *b, struct search *s, } if (!(bio->bi_opf & REQ_RAHEAD) && - !(bio->bi_opf & REQ_META) && + !(bio->bi_opf & REQ_PRIO) && s->iop.c->gc_stats.in_use < CUTOFF_CACHE_READA) reada = min_t(sector_t, dc->readahead >> 9, get_capacity(bio->bi_disk) - bio_end_sector(bio)); @@ -1218,6 +1218,9 @@ static int cached_dev_ioctl(struct bcache_device *d, fmode_t mode, { struct cached_dev *dc = container_of(d, struct cached_dev, disk); + if (dc->io_disable) + return -EIO; + return __blkdev_driver_ioctl(dc->bdev, mode, cmd, arg); } diff --git a/drivers/md/bcache/request.h b/drivers/md/bcache/request.h index aa055cfeb099..721bf336ed1a 100644 --- a/drivers/md/bcache/request.h +++ b/drivers/md/bcache/request.h @@ -39,6 +39,6 @@ void bch_data_insert(struct closure *cl); void bch_cached_dev_request_init(struct cached_dev *dc); void bch_flash_dev_request_init(struct bcache_device *d); -extern struct kmem_cache *bch_search_cache, *bch_passthrough_cache; +extern struct kmem_cache *bch_search_cache; #endif /* _BCACHE_REQUEST_H_ */ diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 30ba9aeb5ee8..7bbd670a5a84 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -418,6 +418,7 @@ static int __uuid_write(struct cache_set *c) { BKEY_PADDED(key) k; struct closure cl; + struct cache *ca; closure_init_stack(&cl); lockdep_assert_held(&bch_register_lock); @@ -429,6 +430,10 @@ static int __uuid_write(struct cache_set *c) uuid_io(c, REQ_OP_WRITE, 0, &k.key, &cl); closure_sync(&cl); + /* Only one bucket used for uuid write */ + ca = PTR_CACHE(c, &k.key, 0); + atomic_long_add(ca->sb.bucket_size, &ca->meta_sectors_written); + bkey_copy(&c->uuid_bucket, &k.key); bkey_put(c, &k.key); return 0; @@ -643,10 +648,6 @@ static int ioctl_dev(struct block_device *b, fmode_t mode, unsigned int cmd, unsigned long arg) { struct bcache_device *d = b->bd_disk->private_data; - struct cached_dev *dc = container_of(d, struct cached_dev, disk); - - if (dc->io_disable) - return -EIO; return d->ioctl(d, mode, cmd, arg); } @@ -1008,6 +1009,7 @@ static void cached_dev_detach_finish(struct work_struct *w) bch_write_bdev_super(dc, &cl); closure_sync(&cl); + calc_cached_dev_sectors(dc->disk.c); bcache_device_detach(&dc->disk); list_move(&dc->list, &uncached_devices); @@ -1152,11 +1154,12 @@ int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c, } if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) { - bch_sectors_dirty_init(&dc->disk); atomic_set(&dc->has_dirty, 1); bch_writeback_queue(dc); } + bch_sectors_dirty_init(&dc->disk); + bch_cached_dev_run(dc); bcache_device_link(&dc->disk, c, "bdev"); atomic_inc(&c->attached_dev_nr); @@ -2049,6 +2052,8 @@ static int cache_alloc(struct cache *ca) size_t free; size_t btree_buckets; struct bucket *b; + int ret = -ENOMEM; + const char *err = NULL; __module_get(THIS_MODULE); kobject_init(&ca->kobj, &bch_cache_ktype); @@ -2066,27 +2071,93 @@ static int cache_alloc(struct cache *ca) */ btree_buckets = ca->sb.njournal_buckets ?: 8; free = roundup_pow_of_two(ca->sb.nbuckets) >> 10; + if (!free) { + ret = -EPERM; + err = "ca->sb.nbuckets is too small"; + goto err_free; + } - if (!init_fifo(&ca->free[RESERVE_BTREE], btree_buckets, GFP_KERNEL) || - !init_fifo_exact(&ca->free[RESERVE_PRIO], prio_buckets(ca), GFP_KERNEL) || - !init_fifo(&ca->free[RESERVE_MOVINGGC], free, GFP_KERNEL) || - !init_fifo(&ca->free[RESERVE_NONE], free, GFP_KERNEL) || - !init_fifo(&ca->free_inc, free << 2, GFP_KERNEL) || - !init_heap(&ca->heap, free << 3, GFP_KERNEL) || - !(ca->buckets = vzalloc(array_size(sizeof(struct bucket), - ca->sb.nbuckets))) || - !(ca->prio_buckets = kzalloc(array3_size(sizeof(uint64_t), - prio_buckets(ca), 2), - GFP_KERNEL)) || - !(ca->disk_buckets = alloc_bucket_pages(GFP_KERNEL, ca))) - return -ENOMEM; + if (!init_fifo(&ca->free[RESERVE_BTREE], btree_buckets, + GFP_KERNEL)) { + err = "ca->free[RESERVE_BTREE] alloc failed"; + goto err_btree_alloc; + } + + if (!init_fifo_exact(&ca->free[RESERVE_PRIO], prio_buckets(ca), + GFP_KERNEL)) { + err = "ca->free[RESERVE_PRIO] alloc failed"; + goto err_prio_alloc; + } + + if (!init_fifo(&ca->free[RESERVE_MOVINGGC], free, GFP_KERNEL)) { + err = "ca->free[RESERVE_MOVINGGC] alloc failed"; + goto err_movinggc_alloc; + } + + if (!init_fifo(&ca->free[RESERVE_NONE], free, GFP_KERNEL)) { + err = "ca->free[RESERVE_NONE] alloc failed"; + goto err_none_alloc; + } + + if (!init_fifo(&ca->free_inc, free << 2, GFP_KERNEL)) { + err = "ca->free_inc alloc failed"; + goto err_free_inc_alloc; + } + + if (!init_heap(&ca->heap, free << 3, GFP_KERNEL)) { + err = "ca->heap alloc failed"; + goto err_heap_alloc; + } + + ca->buckets = vzalloc(array_size(sizeof(struct bucket), + ca->sb.nbuckets)); + if (!ca->buckets) { + err = "ca->buckets alloc failed"; + goto err_buckets_alloc; + } + + ca->prio_buckets = kzalloc(array3_size(sizeof(uint64_t), + prio_buckets(ca), 2), + GFP_KERNEL); + if (!ca->prio_buckets) { + err = "ca->prio_buckets alloc failed"; + goto err_prio_buckets_alloc; + } + + ca->disk_buckets = alloc_bucket_pages(GFP_KERNEL, ca); + if (!ca->disk_buckets) { + err = "ca->disk_buckets alloc failed"; + goto err_disk_buckets_alloc; + } ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca); for_each_bucket(b, ca) atomic_set(&b->pin, 0); - return 0; + +err_disk_buckets_alloc: + kfree(ca->prio_buckets); +err_prio_buckets_alloc: + vfree(ca->buckets); +err_buckets_alloc: + free_heap(&ca->heap); +err_heap_alloc: + free_fifo(&ca->free_inc); +err_free_inc_alloc: + free_fifo(&ca->free[RESERVE_NONE]); +err_none_alloc: + free_fifo(&ca->free[RESERVE_MOVINGGC]); +err_movinggc_alloc: + free_fifo(&ca->free[RESERVE_PRIO]); +err_prio_alloc: + free_fifo(&ca->free[RESERVE_BTREE]); +err_btree_alloc: +err_free: + module_put(THIS_MODULE); + if (err) + pr_notice("error %s: %s", ca->cache_dev_name, err); + return ret; } static int register_cache(struct cache_sb *sb, struct page *sb_page, @@ -2112,6 +2183,8 @@ static int register_cache(struct cache_sb *sb, struct page *sb_page, blkdev_put(bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL); if (ret == -ENOMEM) err = "cache_alloc(): -ENOMEM"; + else if (ret == -EPERM) + err = "cache_alloc(): cache device is too small"; else err = "cache_alloc(): unknown error"; goto err; @@ -2386,7 +2459,7 @@ static int __init bcache_init(void) sysfs_create_files(bcache_kobj, files)) goto err; - bch_debug_init(bcache_kobj); + bch_debug_init(); closure_debug_init(); return 0; diff --git a/drivers/md/bcache/sysfs.c b/drivers/md/bcache/sysfs.c index 150cf4f4cf74..26f035a0c5b9 100644 --- a/drivers/md/bcache/sysfs.c +++ b/drivers/md/bcache/sysfs.c @@ -285,6 +285,7 @@ STORE(__cached_dev) 1, WRITEBACK_RATE_UPDATE_SECS_MAX); d_strtoul(writeback_rate_i_term_inverse); d_strtoul_nonzero(writeback_rate_p_term_inverse); + d_strtoul_nonzero(writeback_rate_minimum); sysfs_strtoul_clamp(io_error_limit, dc->error_limit, 0, INT_MAX); @@ -412,6 +413,7 @@ static struct attribute *bch_cached_dev_files[] = { &sysfs_writeback_rate_update_seconds, &sysfs_writeback_rate_i_term_inverse, &sysfs_writeback_rate_p_term_inverse, + &sysfs_writeback_rate_minimum, &sysfs_writeback_rate_debug, &sysfs_errors, &sysfs_io_error_limit, diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c index ac1cffd2a09b..f3fb5bb8c82a 100644 --- a/drivers/md/raid0.c +++ b/drivers/md/raid0.c @@ -542,7 +542,7 @@ static void raid0_handle_discard(struct mddev *mddev, struct bio *bio) !discard_bio) continue; bio_chain(discard_bio, bio); - bio_clone_blkcg_association(discard_bio, bio); + bio_clone_blkg_association(discard_bio, bio); if (mddev->gendisk) trace_block_bio_remap(bdev_get_queue(rdev->bdev), discard_bio, disk_devt(mddev->gendisk), diff --git a/drivers/memstick/core/ms_block.c b/drivers/memstick/core/ms_block.c index 716fc8ed31d3..8a02f11076f9 100644 --- a/drivers/memstick/core/ms_block.c +++ b/drivers/memstick/core/ms_block.c @@ -2146,7 +2146,7 @@ static int msb_init_disk(struct memstick_dev *card) set_disk_ro(msb->disk, 1); msb_start(card); - device_add_disk(&card->dev, msb->disk); + device_add_disk(&card->dev, msb->disk, NULL); dbg("Disk added"); return 0; diff --git a/drivers/memstick/core/mspro_block.c b/drivers/memstick/core/mspro_block.c index 5ee932631fae..0cd30dcb6801 100644 --- a/drivers/memstick/core/mspro_block.c +++ b/drivers/memstick/core/mspro_block.c @@ -1236,7 +1236,7 @@ static int mspro_block_init_disk(struct memstick_dev *card) set_capacity(msb->disk, capacity); dev_dbg(&card->dev, "capacity set %ld\n", capacity); - device_add_disk(&card->dev, msb->disk); + device_add_disk(&card->dev, msb->disk, NULL); msb->active = 1; return 0; diff --git a/drivers/mfd/altera-a10sr.c b/drivers/mfd/altera-a10sr.c index 96e7d2cb7b89..400e0b51844b 100644 --- a/drivers/mfd/altera-a10sr.c +++ b/drivers/mfd/altera-a10sr.c @@ -108,7 +108,8 @@ static const struct regmap_config altr_a10sr_regmap_config = { .cache_type = REGCACHE_NONE, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, .read_flag_mask = 1, .write_flag_mask = 0, diff --git a/drivers/mfd/da9052-spi.c b/drivers/mfd/da9052-spi.c index abfb11818fdc..fdae1288bc6d 100644 --- a/drivers/mfd/da9052-spi.c +++ b/drivers/mfd/da9052-spi.c @@ -46,7 +46,8 @@ static int da9052_spi_probe(struct spi_device *spi) config.reg_bits = 7; config.pad_bits = 1; config.val_bits = 8; - config.use_single_rw = 1; + config.use_single_read = true; + config.use_single_write = true; da9052->regmap = devm_regmap_init_spi(spi, &config); if (IS_ERR(da9052->regmap)) { diff --git a/drivers/mfd/mc13xxx-spi.c b/drivers/mfd/mc13xxx-spi.c index cbc1e5ed599c..ee3411cc5ce4 100644 --- a/drivers/mfd/mc13xxx-spi.c +++ b/drivers/mfd/mc13xxx-spi.c @@ -57,7 +57,8 @@ static const struct regmap_config mc13xxx_regmap_spi_config = { .max_register = MC13XXX_NUMREGS, .cache_type = REGCACHE_NONE, - .use_single_rw = 1, + .use_single_read = true, + .use_single_write = true, }; static int mc13xxx_spi_read(void *context, const void *reg, size_t reg_size, diff --git a/drivers/mfd/twl6040.c b/drivers/mfd/twl6040.c index dd19f17a1b63..7c3c5fd5fcd0 100644 --- a/drivers/mfd/twl6040.c +++ b/drivers/mfd/twl6040.c @@ -613,7 +613,8 @@ static const struct regmap_config twl6040_regmap_config = { .writeable_reg = twl6040_writeable_reg, .cache_type = REGCACHE_RBTREE, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; static const struct regmap_irq twl6040_irqs[] = { diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index e201ccb3fda4..c35b5b08bb33 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -2708,7 +2708,7 @@ static int mmc_add_disk(struct mmc_blk_data *md) int ret; struct mmc_card *card = md->queue.card; - device_add_disk(md->parent, md->disk); + device_add_disk(md->parent, md->disk, NULL); md->force_ro.show = force_ro_show; md->force_ro.store = force_ro_store; sysfs_attr_init(&md->force_ro.attr); diff --git a/drivers/mtd/devices/m25p80.c b/drivers/mtd/devices/m25p80.c index 270d3c9580c5..c4a1d04b8c80 100644 --- a/drivers/mtd/devices/m25p80.c +++ b/drivers/mtd/devices/m25p80.c @@ -90,7 +90,6 @@ static ssize_t m25p80_write(struct spi_nor *nor, loff_t to, size_t len, SPI_MEM_OP_ADDR(nor->addr_width, to, 1), SPI_MEM_OP_NO_DUMMY, SPI_MEM_OP_DATA_OUT(len, buf, 1)); - size_t remaining = len; int ret; /* get transfer protocols. */ @@ -101,22 +100,16 @@ static ssize_t m25p80_write(struct spi_nor *nor, loff_t to, size_t len, if (nor->program_opcode == SPINOR_OP_AAI_WP && nor->sst_write_second) op.addr.nbytes = 0; - while (remaining) { - op.data.nbytes = remaining < UINT_MAX ? remaining : UINT_MAX; - ret = spi_mem_adjust_op_size(flash->spimem, &op); - if (ret) - return ret; - - ret = spi_mem_exec_op(flash->spimem, &op); - if (ret) - return ret; + ret = spi_mem_adjust_op_size(flash->spimem, &op); + if (ret) + return ret; + op.data.nbytes = len < op.data.nbytes ? len : op.data.nbytes; - op.addr.val += op.data.nbytes; - remaining -= op.data.nbytes; - op.data.buf.out += op.data.nbytes; - } + ret = spi_mem_exec_op(flash->spimem, &op); + if (ret) + return ret; - return len; + return op.data.nbytes; } /* diff --git a/drivers/mtd/maps/gpio-addr-flash.c b/drivers/mtd/maps/gpio-addr-flash.c index 9d9723693217..a20e85aa770e 100644 --- a/drivers/mtd/maps/gpio-addr-flash.c +++ b/drivers/mtd/maps/gpio-addr-flash.c @@ -14,6 +14,7 @@ */ #include <linux/gpio.h> +#include <linux/gpio/consumer.h> #include <linux/io.h> #include <linux/kernel.h> #include <linux/module.h> @@ -25,28 +26,24 @@ #include <linux/slab.h> #include <linux/types.h> -#define pr_devinit(fmt, args...) \ - ({ static const char __fmt[] = fmt; printk(__fmt, ## args); }) +#define win_mask(x) ((BIT(x)) - 1) #define DRIVER_NAME "gpio-addr-flash" -#define PFX DRIVER_NAME ": " /** * struct async_state - keep GPIO flash state * @mtd: MTD state for this mapping * @map: MTD map state for this flash - * @gpio_count: number of GPIOs used to address - * @gpio_addrs: array of GPIOs to twiddle + * @gpios: Struct containing the array of GPIO descriptors * @gpio_values: cached GPIO values - * @win_size: dedicated memory size (if no GPIOs) + * @win_order: dedicated memory size (if no GPIOs) */ struct async_state { struct mtd_info *mtd; struct map_info map; - size_t gpio_count; - unsigned *gpio_addrs; - int *gpio_values; - unsigned long win_size; + struct gpio_descs *gpios; + unsigned int gpio_values; + unsigned int win_order; }; #define gf_map_info_to_state(mi) ((struct async_state *)(mi)->map_priv_1) @@ -57,21 +54,25 @@ struct async_state { * * Rather than call the GPIO framework every time, cache the last-programmed * value. This speeds up sequential accesses (which are by far the most common - * type). We rely on the GPIO framework to treat non-zero value as high so - * that we don't have to normalize the bits. + * type). */ static void gf_set_gpios(struct async_state *state, unsigned long ofs) { - size_t i = 0; - int value; - ofs /= state->win_size; - do { - value = ofs & (1 << i); - if (state->gpio_values[i] != value) { - gpio_set_value(state->gpio_addrs[i], value); - state->gpio_values[i] = value; - } - } while (++i < state->gpio_count); + int i; + + ofs >>= state->win_order; + + if (ofs == state->gpio_values) + return; + + for (i = 0; i < state->gpios->ndescs; i++) { + if ((ofs & BIT(i)) == (state->gpio_values & BIT(i))) + continue; + + gpiod_set_value(state->gpios->desc[i], !!(ofs & BIT(i))); + } + + state->gpio_values = ofs; } /** @@ -87,7 +88,7 @@ static map_word gf_read(struct map_info *map, unsigned long ofs) gf_set_gpios(state, ofs); - word = readw(map->virt + (ofs % state->win_size)); + word = readw(map->virt + (ofs & win_mask(state->win_order))); test.x[0] = word; return test; } @@ -109,14 +110,14 @@ static void gf_copy_from(struct map_info *map, void *to, unsigned long from, ssi int this_len; while (len) { - if ((from % state->win_size) + len > state->win_size) - this_len = state->win_size - (from % state->win_size); - else - this_len = len; + this_len = from & win_mask(state->win_order); + this_len = BIT(state->win_order) - this_len; + this_len = min_t(int, len, this_len); gf_set_gpios(state, from); - memcpy_fromio(to, map->virt + (from % state->win_size), - this_len); + memcpy_fromio(to, + map->virt + (from & win_mask(state->win_order)), + this_len); len -= this_len; from += this_len; to += this_len; @@ -136,7 +137,7 @@ static void gf_write(struct map_info *map, map_word d1, unsigned long ofs) gf_set_gpios(state, ofs); d = d1.x[0]; - writew(d, map->virt + (ofs % state->win_size)); + writew(d, map->virt + (ofs & win_mask(state->win_order))); } /** @@ -156,13 +157,13 @@ static void gf_copy_to(struct map_info *map, unsigned long to, int this_len; while (len) { - if ((to % state->win_size) + len > state->win_size) - this_len = state->win_size - (to % state->win_size); - else - this_len = len; + this_len = to & win_mask(state->win_order); + this_len = BIT(state->win_order) - this_len; + this_len = min_t(int, len, this_len); gf_set_gpios(state, to); - memcpy_toio(map->virt + (to % state->win_size), from, len); + memcpy_toio(map->virt + (to & win_mask(state->win_order)), + from, len); len -= this_len; to += this_len; @@ -180,18 +181,22 @@ static const char * const part_probe_types[] = { * The platform resource layout expected looks something like: * struct mtd_partition partitions[] = { ... }; * struct physmap_flash_data flash_data = { ... }; - * unsigned flash_gpios[] = { GPIO_XX, GPIO_XX, ... }; + * static struct gpiod_lookup_table addr_flash_gpios = { + * .dev_id = "gpio-addr-flash.0", + * .table = { + * GPIO_LOOKUP_IDX("gpio.0", 15, "addr", 0, GPIO_ACTIVE_HIGH), + * GPIO_LOOKUP_IDX("gpio.0", 16, "addr", 1, GPIO_ACTIVE_HIGH), + * ); + * }; + * gpiod_add_lookup_table(&addr_flash_gpios); + * * struct resource flash_resource[] = { * { * .name = "cfi_probe", * .start = 0x20000000, * .end = 0x201fffff, * .flags = IORESOURCE_MEM, - * }, { - * .start = (unsigned long)flash_gpios, - * .end = ARRAY_SIZE(flash_gpios), - * .flags = IORESOURCE_IRQ, - * } + * }, * }; * struct platform_device flash_device = { * .name = "gpio-addr-flash", @@ -203,33 +208,25 @@ static const char * const part_probe_types[] = { */ static int gpio_flash_probe(struct platform_device *pdev) { - size_t i, arr_size; struct physmap_flash_data *pdata; struct resource *memory; - struct resource *gpios; struct async_state *state; pdata = dev_get_platdata(&pdev->dev); memory = platform_get_resource(pdev, IORESOURCE_MEM, 0); - gpios = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!memory || !gpios || !gpios->end) + if (!memory) return -EINVAL; - arr_size = sizeof(int) * gpios->end; - state = kzalloc(sizeof(*state) + arr_size, GFP_KERNEL); + state = devm_kzalloc(&pdev->dev, sizeof(*state), GFP_KERNEL); if (!state) return -ENOMEM; - /* - * We cast start/end to known types in the boards file, so cast - * away their pointer types here to the known types (gpios->xxx). - */ - state->gpio_count = gpios->end; - state->gpio_addrs = (void *)(unsigned long)gpios->start; - state->gpio_values = (void *)(state + 1); - state->win_size = resource_size(memory); - memset(state->gpio_values, 0xff, arr_size); + state->gpios = devm_gpiod_get_array(&pdev->dev, "addr", GPIOD_OUT_LOW); + if (IS_ERR(state->gpios)) + return PTR_ERR(state->gpios); + + state->win_order = get_bitmask_order(resource_size(memory)) - 1; state->map.name = DRIVER_NAME; state->map.read = gf_read; @@ -237,38 +234,21 @@ static int gpio_flash_probe(struct platform_device *pdev) state->map.write = gf_write; state->map.copy_to = gf_copy_to; state->map.bankwidth = pdata->width; - state->map.size = state->win_size * (1 << state->gpio_count); - state->map.virt = ioremap_nocache(memory->start, state->map.size); - if (!state->map.virt) - return -ENOMEM; + state->map.size = BIT(state->win_order + state->gpios->ndescs); + state->map.virt = devm_ioremap_resource(&pdev->dev, memory); + if (IS_ERR(state->map.virt)) + return PTR_ERR(state->map.virt); state->map.phys = NO_XIP; state->map.map_priv_1 = (unsigned long)state; platform_set_drvdata(pdev, state); - i = 0; - do { - if (gpio_request(state->gpio_addrs[i], DRIVER_NAME)) { - pr_devinit(KERN_ERR PFX "failed to request gpio %d\n", - state->gpio_addrs[i]); - while (i--) - gpio_free(state->gpio_addrs[i]); - kfree(state); - return -EBUSY; - } - gpio_direction_output(state->gpio_addrs[i], 0); - } while (++i < state->gpio_count); - - pr_devinit(KERN_NOTICE PFX "probing %d-bit flash bus\n", - state->map.bankwidth * 8); + dev_notice(&pdev->dev, "probing %d-bit flash bus\n", + state->map.bankwidth * 8); state->mtd = do_map_probe(memory->name, &state->map); - if (!state->mtd) { - for (i = 0; i < state->gpio_count; ++i) - gpio_free(state->gpio_addrs[i]); - kfree(state); + if (!state->mtd) return -ENXIO; - } state->mtd->dev.parent = &pdev->dev; mtd_device_parse_register(state->mtd, part_probe_types, NULL, @@ -280,13 +260,9 @@ static int gpio_flash_probe(struct platform_device *pdev) static int gpio_flash_remove(struct platform_device *pdev) { struct async_state *state = platform_get_drvdata(pdev); - size_t i = 0; - do { - gpio_free(state->gpio_addrs[i]); - } while (++i < state->gpio_count); + mtd_device_unregister(state->mtd); map_destroy(state->mtd); - kfree(state); return 0; } diff --git a/drivers/mtd/maps/physmap_of_core.c b/drivers/mtd/maps/physmap_of_core.c index 4129535b8e46..ece605d78c21 100644 --- a/drivers/mtd/maps/physmap_of_core.c +++ b/drivers/mtd/maps/physmap_of_core.c @@ -31,7 +31,6 @@ struct of_flash_list { struct mtd_info *mtd; struct map_info map; - struct resource *res; }; struct of_flash { @@ -56,18 +55,10 @@ static int of_flash_remove(struct platform_device *dev) mtd_concat_destroy(info->cmtd); } - for (i = 0; i < info->list_size; i++) { + for (i = 0; i < info->list_size; i++) if (info->list[i].mtd) map_destroy(info->list[i].mtd); - if (info->list[i].map.virt) - iounmap(info->list[i].map.virt); - - if (info->list[i].res) { - release_resource(info->list[i].res); - kfree(info->list[i].res); - } - } return 0; } @@ -215,10 +206,11 @@ static int of_flash_probe(struct platform_device *dev) err = -EBUSY; res_size = resource_size(&res); - info->list[i].res = request_mem_region(res.start, res_size, - dev_name(&dev->dev)); - if (!info->list[i].res) + info->list[i].map.virt = devm_ioremap_resource(&dev->dev, &res); + if (IS_ERR(info->list[i].map.virt)) { + err = PTR_ERR(info->list[i].map.virt); goto err_out; + } err = -ENXIO; width = of_get_property(dp, "bank-width", NULL); @@ -246,15 +238,6 @@ static int of_flash_probe(struct platform_device *dev) if (err) goto err_out; - err = -ENOMEM; - info->list[i].map.virt = ioremap(info->list[i].map.phys, - info->list[i].map.size); - if (!info->list[i].map.virt) { - dev_err(&dev->dev, "Failed to ioremap() flash" - " region\n"); - goto err_out; - } - simple_map_init(&info->list[i].map); /* diff --git a/drivers/mtd/maps/physmap_of_gemini.c b/drivers/mtd/maps/physmap_of_gemini.c index 830b1b7e702b..9df62ca721d5 100644 --- a/drivers/mtd/maps/physmap_of_gemini.c +++ b/drivers/mtd/maps/physmap_of_gemini.c @@ -44,11 +44,6 @@ #define FLASH_PARALLEL_HIGH_PIN_CNT (1 << 20) /* else low pin cnt */ -static const struct of_device_id syscon_match[] = { - { .compatible = "cortina,gemini-syscon" }, - { }, -}; - int of_flash_probe_gemini(struct platform_device *pdev, struct device_node *np, struct map_info *map) diff --git a/drivers/mtd/mtd_blkdevs.c b/drivers/mtd/mtd_blkdevs.c index 29c0bfd74e8a..b0d44f9214b0 100644 --- a/drivers/mtd/mtd_blkdevs.c +++ b/drivers/mtd/mtd_blkdevs.c @@ -27,6 +27,7 @@ #include <linux/mtd/blktrans.h> #include <linux/mtd/mtd.h> #include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/blkpg.h> #include <linux/spinlock.h> #include <linux/hdreg.h> @@ -45,6 +46,8 @@ static void blktrans_dev_release(struct kref *kref) dev->disk->private_data = NULL; blk_cleanup_queue(dev->rq); + blk_mq_free_tag_set(dev->tag_set); + kfree(dev->tag_set); put_disk(dev->disk); list_del(&dev->list); kfree(dev); @@ -134,28 +137,39 @@ int mtd_blktrans_cease_background(struct mtd_blktrans_dev *dev) } EXPORT_SYMBOL_GPL(mtd_blktrans_cease_background); -static void mtd_blktrans_work(struct work_struct *work) +static struct request *mtd_next_request(struct mtd_blktrans_dev *dev) +{ + struct request *rq; + + rq = list_first_entry_or_null(&dev->rq_list, struct request, queuelist); + if (rq) { + list_del_init(&rq->queuelist); + blk_mq_start_request(rq); + return rq; + } + + return NULL; +} + +static void mtd_blktrans_work(struct mtd_blktrans_dev *dev) + __releases(&dev->queue_lock) + __acquires(&dev->queue_lock) { - struct mtd_blktrans_dev *dev = - container_of(work, struct mtd_blktrans_dev, work); struct mtd_blktrans_ops *tr = dev->tr; - struct request_queue *rq = dev->rq; struct request *req = NULL; int background_done = 0; - spin_lock_irq(rq->queue_lock); - while (1) { blk_status_t res; dev->bg_stop = false; - if (!req && !(req = blk_fetch_request(rq))) { + if (!req && !(req = mtd_next_request(dev))) { if (tr->background && !background_done) { - spin_unlock_irq(rq->queue_lock); + spin_unlock_irq(&dev->queue_lock); mutex_lock(&dev->lock); tr->background(dev); mutex_unlock(&dev->lock); - spin_lock_irq(rq->queue_lock); + spin_lock_irq(&dev->queue_lock); /* * Do background processing just once per idle * period. @@ -166,35 +180,39 @@ static void mtd_blktrans_work(struct work_struct *work) break; } - spin_unlock_irq(rq->queue_lock); + spin_unlock_irq(&dev->queue_lock); mutex_lock(&dev->lock); res = do_blktrans_request(dev->tr, dev, req); mutex_unlock(&dev->lock); - spin_lock_irq(rq->queue_lock); - - if (!__blk_end_request_cur(req, res)) + if (!blk_update_request(req, res, blk_rq_cur_bytes(req))) { + __blk_mq_end_request(req, res); req = NULL; + } background_done = 0; + spin_lock_irq(&dev->queue_lock); } - - spin_unlock_irq(rq->queue_lock); } -static void mtd_blktrans_request(struct request_queue *rq) +static blk_status_t mtd_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) { struct mtd_blktrans_dev *dev; - struct request *req = NULL; - dev = rq->queuedata; + dev = hctx->queue->queuedata; + if (!dev) { + blk_mq_start_request(bd->rq); + return BLK_STS_IOERR; + } + + spin_lock_irq(&dev->queue_lock); + list_add_tail(&bd->rq->queuelist, &dev->rq_list); + mtd_blktrans_work(dev); + spin_unlock_irq(&dev->queue_lock); - if (!dev) - while ((req = blk_fetch_request(rq)) != NULL) - __blk_end_request_all(req, BLK_STS_IOERR); - else - queue_work(dev->wq, &dev->work); + return BLK_STS_OK; } static int blktrans_open(struct block_device *bdev, fmode_t mode) @@ -329,6 +347,10 @@ static const struct block_device_operations mtd_block_ops = { .getgeo = blktrans_getgeo, }; +static const struct blk_mq_ops mtd_mq_ops = { + .queue_rq = mtd_queue_rq, +}; + int add_mtd_blktrans_dev(struct mtd_blktrans_dev *new) { struct mtd_blktrans_ops *tr = new->tr; @@ -416,11 +438,20 @@ int add_mtd_blktrans_dev(struct mtd_blktrans_dev *new) /* Create the request queue */ spin_lock_init(&new->queue_lock); - new->rq = blk_init_queue(mtd_blktrans_request, &new->queue_lock); + INIT_LIST_HEAD(&new->rq_list); - if (!new->rq) + new->tag_set = kzalloc(sizeof(*new->tag_set), GFP_KERNEL); + if (!new->tag_set) goto error3; + new->rq = blk_mq_init_sq_queue(new->tag_set, &mtd_mq_ops, 2, + BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING); + if (IS_ERR(new->rq)) { + ret = PTR_ERR(new->rq); + new->rq = NULL; + goto error4; + } + if (tr->flush) blk_queue_write_cache(new->rq, true, false); @@ -437,17 +468,10 @@ int add_mtd_blktrans_dev(struct mtd_blktrans_dev *new) gd->queue = new->rq; - /* Create processing workqueue */ - new->wq = alloc_workqueue("%s%d", 0, 0, - tr->name, new->mtd->index); - if (!new->wq) - goto error4; - INIT_WORK(&new->work, mtd_blktrans_work); - if (new->readonly) set_disk_ro(gd, 1); - device_add_disk(&new->mtd->dev, gd); + device_add_disk(&new->mtd->dev, gd, NULL); if (new->disk_attributes) { ret = sysfs_create_group(&disk_to_dev(gd)->kobj, @@ -456,7 +480,7 @@ int add_mtd_blktrans_dev(struct mtd_blktrans_dev *new) } return 0; error4: - blk_cleanup_queue(new->rq); + kfree(new->tag_set); error3: put_disk(new->disk); error2: @@ -481,15 +505,17 @@ int del_mtd_blktrans_dev(struct mtd_blktrans_dev *old) /* Stop new requests to arrive */ del_gendisk(old->disk); - /* Stop workqueue. This will perform any pending request. */ - destroy_workqueue(old->wq); - /* Kill current requests */ spin_lock_irqsave(&old->queue_lock, flags); old->rq->queuedata = NULL; - blk_start_queue(old->rq); spin_unlock_irqrestore(&old->queue_lock, flags); + /* freeze+quiesce queue to ensure all requests are flushed */ + blk_mq_freeze_queue(old->rq); + blk_mq_quiesce_queue(old->rq); + blk_mq_unquiesce_queue(old->rq); + blk_mq_unfreeze_queue(old->rq); + /* If the device is currently open, tell trans driver to close it, then put mtd device, and don't touch it again */ mutex_lock(&old->lock); diff --git a/drivers/mtd/nand/raw/Kconfig b/drivers/mtd/nand/raw/Kconfig index 5fc9a1bde4ac..c7efc31384d5 100644 --- a/drivers/mtd/nand/raw/Kconfig +++ b/drivers/mtd/nand/raw/Kconfig @@ -227,26 +227,6 @@ config MTD_NAND_DISKONCHIP_BBTWRITE load time (assuming you build diskonchip as a module) with the module parameter "inftl_bbt_write=1". -config MTD_NAND_DOCG4 - tristate "Support for DiskOnChip G4" - depends on HAS_IOMEM - select BCH - select BITREVERSE - help - Support for diskonchip G4 nand flash, found in various smartphones and - PDAs, among them the Palm Treo680, HTC Prophet and Wizard, Toshiba - Portege G900, Asus P526, and O2 XDA Zinc. - - With this driver you will be able to use UBI and create a ubifs on the - device, so you may wish to consider enabling UBI and UBIFS as well. - - These devices ship with the Mys/Sandisk SAFTL formatting, for which - there is currently no mtd parser, so you may want to use command line - partitioning to segregate write-protected blocks. On the Treo680, the - first five erase blocks (256KiB each) are write-protected, followed - by the block containing the saftl partition table. This is probably - typical. - config MTD_NAND_SHARPSL tristate "Support for NAND Flash on Sharp SL Series (C7xx + others)" depends on ARCH_PXA || COMPILE_TEST diff --git a/drivers/mtd/nand/raw/Makefile b/drivers/mtd/nand/raw/Makefile index d5a5f9832b88..57159b349054 100644 --- a/drivers/mtd/nand/raw/Makefile +++ b/drivers/mtd/nand/raw/Makefile @@ -15,7 +15,6 @@ obj-$(CONFIG_MTD_NAND_S3C2410) += s3c2410.o obj-$(CONFIG_MTD_NAND_TANGO) += tango_nand.o obj-$(CONFIG_MTD_NAND_DAVINCI) += davinci_nand.o obj-$(CONFIG_MTD_NAND_DISKONCHIP) += diskonchip.o -obj-$(CONFIG_MTD_NAND_DOCG4) += docg4.o obj-$(CONFIG_MTD_NAND_FSMC) += fsmc_nand.o obj-$(CONFIG_MTD_NAND_SHARPSL) += sharpsl.o obj-$(CONFIG_MTD_NAND_NANDSIM) += nandsim.o @@ -58,8 +57,11 @@ obj-$(CONFIG_MTD_NAND_QCOM) += qcom_nandc.o obj-$(CONFIG_MTD_NAND_MTK) += mtk_ecc.o mtk_nand.o obj-$(CONFIG_MTD_NAND_TEGRA) += tegra_nand.o -nand-objs := nand_base.o nand_bbt.o nand_timings.o nand_ids.o +nand-objs := nand_base.o nand_legacy.o nand_bbt.o nand_timings.o nand_ids.o +nand-objs += nand_onfi.o +nand-objs += nand_jedec.o nand-objs += nand_amd.o +nand-objs += nand_esmt.o nand-objs += nand_hynix.o nand-objs += nand_macronix.o nand-objs += nand_micron.o diff --git a/drivers/mtd/nand/raw/ams-delta.c b/drivers/mtd/nand/raw/ams-delta.c index 37a3cc21c7bc..5ba180a291eb 100644 --- a/drivers/mtd/nand/raw/ams-delta.c +++ b/drivers/mtd/nand/raw/ams-delta.c @@ -20,23 +20,33 @@ #include <linux/slab.h> #include <linux/module.h> #include <linux/delay.h> +#include <linux/gpio/consumer.h> #include <linux/mtd/mtd.h> #include <linux/mtd/rawnand.h> #include <linux/mtd/partitions.h> -#include <linux/gpio.h> #include <linux/platform_data/gpio-omap.h> #include <asm/io.h> #include <asm/sizes.h> -#include <mach/board-ams-delta.h> - #include <mach/hardware.h> /* * MTD structure for E3 (Delta) */ -static struct mtd_info *ams_delta_mtd = NULL; + +struct ams_delta_nand { + struct nand_chip nand_chip; + struct gpio_desc *gpiod_rdy; + struct gpio_desc *gpiod_nce; + struct gpio_desc *gpiod_nre; + struct gpio_desc *gpiod_nwp; + struct gpio_desc *gpiod_nwe; + struct gpio_desc *gpiod_ale; + struct gpio_desc *gpiod_cle; + void __iomem *io_base; + bool data_in; +}; /* * Define partitions for flash devices @@ -63,48 +73,64 @@ static const struct mtd_partition partition_info[] = { .size = 3 * SZ_256K }, }; -static void ams_delta_write_byte(struct mtd_info *mtd, u_char byte) +static void ams_delta_io_write(struct ams_delta_nand *priv, u_char byte) { - struct nand_chip *this = mtd_to_nand(mtd); - void __iomem *io_base = (void __iomem *)nand_get_controller_data(this); - - writew(0, io_base + OMAP_MPUIO_IO_CNTL); - writew(byte, this->IO_ADDR_W); - gpio_set_value(AMS_DELTA_GPIO_PIN_NAND_NWE, 0); + writew(byte, priv->nand_chip.legacy.IO_ADDR_W); + gpiod_set_value(priv->gpiod_nwe, 0); ndelay(40); - gpio_set_value(AMS_DELTA_GPIO_PIN_NAND_NWE, 1); + gpiod_set_value(priv->gpiod_nwe, 1); } -static u_char ams_delta_read_byte(struct mtd_info *mtd) +static u_char ams_delta_io_read(struct ams_delta_nand *priv) { u_char res; - struct nand_chip *this = mtd_to_nand(mtd); - void __iomem *io_base = (void __iomem *)nand_get_controller_data(this); - gpio_set_value(AMS_DELTA_GPIO_PIN_NAND_NRE, 0); + gpiod_set_value(priv->gpiod_nre, 0); ndelay(40); - writew(~0, io_base + OMAP_MPUIO_IO_CNTL); - res = readw(this->IO_ADDR_R); - gpio_set_value(AMS_DELTA_GPIO_PIN_NAND_NRE, 1); + res = readw(priv->nand_chip.legacy.IO_ADDR_R); + gpiod_set_value(priv->gpiod_nre, 1); return res; } -static void ams_delta_write_buf(struct mtd_info *mtd, const u_char *buf, +static void ams_delta_dir_input(struct ams_delta_nand *priv, bool in) +{ + writew(in ? ~0 : 0, priv->io_base + OMAP_MPUIO_IO_CNTL); + priv->data_in = in; +} + +static void ams_delta_write_buf(struct nand_chip *this, const u_char *buf, int len) { + struct ams_delta_nand *priv = nand_get_controller_data(this); int i; - for (i=0; i<len; i++) - ams_delta_write_byte(mtd, buf[i]); + if (priv->data_in) + ams_delta_dir_input(priv, false); + + for (i = 0; i < len; i++) + ams_delta_io_write(priv, buf[i]); } -static void ams_delta_read_buf(struct mtd_info *mtd, u_char *buf, int len) +static void ams_delta_read_buf(struct nand_chip *this, u_char *buf, int len) { + struct ams_delta_nand *priv = nand_get_controller_data(this); int i; - for (i=0; i<len; i++) - buf[i] = ams_delta_read_byte(mtd); + if (!priv->data_in) + ams_delta_dir_input(priv, true); + + for (i = 0; i < len; i++) + buf[i] = ams_delta_io_read(priv); +} + +static u_char ams_delta_read_byte(struct nand_chip *this) +{ + u_char res; + + ams_delta_read_buf(this, &res, 1); + + return res; } /* @@ -115,67 +141,40 @@ static void ams_delta_read_buf(struct mtd_info *mtd, u_char *buf, int len) * NAND_CLE: bit 1 -> bit 7 * NAND_ALE: bit 2 -> bit 6 */ -static void ams_delta_hwcontrol(struct mtd_info *mtd, int cmd, +static void ams_delta_hwcontrol(struct nand_chip *this, int cmd, unsigned int ctrl) { + struct ams_delta_nand *priv = nand_get_controller_data(this); if (ctrl & NAND_CTRL_CHANGE) { - gpio_set_value(AMS_DELTA_GPIO_PIN_NAND_NCE, - (ctrl & NAND_NCE) == 0); - gpio_set_value(AMS_DELTA_GPIO_PIN_NAND_CLE, - (ctrl & NAND_CLE) != 0); - gpio_set_value(AMS_DELTA_GPIO_PIN_NAND_ALE, - (ctrl & NAND_ALE) != 0); + gpiod_set_value(priv->gpiod_nce, !(ctrl & NAND_NCE)); + gpiod_set_value(priv->gpiod_cle, !!(ctrl & NAND_CLE)); + gpiod_set_value(priv->gpiod_ale, !!(ctrl & NAND_ALE)); } - if (cmd != NAND_CMD_NONE) - ams_delta_write_byte(mtd, cmd); + if (cmd != NAND_CMD_NONE) { + u_char byte = cmd; + + ams_delta_write_buf(this, &byte, 1); + } } -static int ams_delta_nand_ready(struct mtd_info *mtd) +static int ams_delta_nand_ready(struct nand_chip *this) { - return gpio_get_value(AMS_DELTA_GPIO_PIN_NAND_RB); + struct ams_delta_nand *priv = nand_get_controller_data(this); + + return gpiod_get_value(priv->gpiod_rdy); } -static const struct gpio _mandatory_gpio[] = { - { - .gpio = AMS_DELTA_GPIO_PIN_NAND_NCE, - .flags = GPIOF_OUT_INIT_HIGH, - .label = "nand_nce", - }, - { - .gpio = AMS_DELTA_GPIO_PIN_NAND_NRE, - .flags = GPIOF_OUT_INIT_HIGH, - .label = "nand_nre", - }, - { - .gpio = AMS_DELTA_GPIO_PIN_NAND_NWP, - .flags = GPIOF_OUT_INIT_HIGH, - .label = "nand_nwp", - }, - { - .gpio = AMS_DELTA_GPIO_PIN_NAND_NWE, - .flags = GPIOF_OUT_INIT_HIGH, - .label = "nand_nwe", - }, - { - .gpio = AMS_DELTA_GPIO_PIN_NAND_ALE, - .flags = GPIOF_OUT_INIT_LOW, - .label = "nand_ale", - }, - { - .gpio = AMS_DELTA_GPIO_PIN_NAND_CLE, - .flags = GPIOF_OUT_INIT_LOW, - .label = "nand_cle", - }, -}; /* * Main initialization routine */ static int ams_delta_init(struct platform_device *pdev) { + struct ams_delta_nand *priv; struct nand_chip *this; + struct mtd_info *mtd; struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0); void __iomem *io_base; int err = 0; @@ -184,15 +183,16 @@ static int ams_delta_init(struct platform_device *pdev) return -ENXIO; /* Allocate memory for MTD device structure and private data */ - this = kzalloc(sizeof(struct nand_chip), GFP_KERNEL); - if (!this) { + priv = devm_kzalloc(&pdev->dev, sizeof(struct ams_delta_nand), + GFP_KERNEL); + if (!priv) { pr_warn("Unable to allocate E3 NAND MTD device structure.\n"); - err = -ENOMEM; - goto out; + return -ENOMEM; } + this = &priv->nand_chip; - ams_delta_mtd = nand_to_mtd(this); - ams_delta_mtd->owner = THIS_MODULE; + mtd = nand_to_mtd(this); + mtd->dev.parent = &pdev->dev; /* * Don't try to request the memory region from here, @@ -207,51 +207,93 @@ static int ams_delta_init(struct platform_device *pdev) goto out_free; } - nand_set_controller_data(this, (void *)io_base); + priv->io_base = io_base; + nand_set_controller_data(this, priv); /* Set address of NAND IO lines */ - this->IO_ADDR_R = io_base + OMAP_MPUIO_INPUT_LATCH; - this->IO_ADDR_W = io_base + OMAP_MPUIO_OUTPUT; - this->read_byte = ams_delta_read_byte; - this->write_buf = ams_delta_write_buf; - this->read_buf = ams_delta_read_buf; - this->cmd_ctrl = ams_delta_hwcontrol; - if (gpio_request(AMS_DELTA_GPIO_PIN_NAND_RB, "nand_rdy") == 0) { - this->dev_ready = ams_delta_nand_ready; - } else { - this->dev_ready = NULL; - pr_notice("Couldn't request gpio for Delta NAND ready.\n"); + this->legacy.IO_ADDR_R = io_base + OMAP_MPUIO_INPUT_LATCH; + this->legacy.IO_ADDR_W = io_base + OMAP_MPUIO_OUTPUT; + this->legacy.read_byte = ams_delta_read_byte; + this->legacy.write_buf = ams_delta_write_buf; + this->legacy.read_buf = ams_delta_read_buf; + this->legacy.cmd_ctrl = ams_delta_hwcontrol; + + priv->gpiod_rdy = devm_gpiod_get_optional(&pdev->dev, "rdy", GPIOD_IN); + if (IS_ERR(priv->gpiod_rdy)) { + err = PTR_ERR(priv->gpiod_rdy); + dev_warn(&pdev->dev, "RDY GPIO request failed (%d)\n", err); + goto out_mtd; } + + if (priv->gpiod_rdy) + this->legacy.dev_ready = ams_delta_nand_ready; + /* 25 us command delay time */ - this->chip_delay = 30; + this->legacy.chip_delay = 30; this->ecc.mode = NAND_ECC_SOFT; this->ecc.algo = NAND_ECC_HAMMING; - platform_set_drvdata(pdev, io_base); + platform_set_drvdata(pdev, priv); /* Set chip enabled, but */ - err = gpio_request_array(_mandatory_gpio, ARRAY_SIZE(_mandatory_gpio)); - if (err) - goto out_gpio; + priv->gpiod_nwp = devm_gpiod_get(&pdev->dev, "nwp", GPIOD_OUT_HIGH); + if (IS_ERR(priv->gpiod_nwp)) { + err = PTR_ERR(priv->gpiod_nwp); + dev_err(&pdev->dev, "NWP GPIO request failed (%d)\n", err); + goto out_mtd; + } + + priv->gpiod_nce = devm_gpiod_get(&pdev->dev, "nce", GPIOD_OUT_HIGH); + if (IS_ERR(priv->gpiod_nce)) { + err = PTR_ERR(priv->gpiod_nce); + dev_err(&pdev->dev, "NCE GPIO request failed (%d)\n", err); + goto out_mtd; + } + + priv->gpiod_nre = devm_gpiod_get(&pdev->dev, "nre", GPIOD_OUT_HIGH); + if (IS_ERR(priv->gpiod_nre)) { + err = PTR_ERR(priv->gpiod_nre); + dev_err(&pdev->dev, "NRE GPIO request failed (%d)\n", err); + goto out_mtd; + } + + priv->gpiod_nwe = devm_gpiod_get(&pdev->dev, "nwe", GPIOD_OUT_HIGH); + if (IS_ERR(priv->gpiod_nwe)) { + err = PTR_ERR(priv->gpiod_nwe); + dev_err(&pdev->dev, "NWE GPIO request failed (%d)\n", err); + goto out_mtd; + } + + priv->gpiod_ale = devm_gpiod_get(&pdev->dev, "ale", GPIOD_OUT_LOW); + if (IS_ERR(priv->gpiod_ale)) { + err = PTR_ERR(priv->gpiod_ale); + dev_err(&pdev->dev, "ALE GPIO request failed (%d)\n", err); + goto out_mtd; + } + + priv->gpiod_cle = devm_gpiod_get(&pdev->dev, "cle", GPIOD_OUT_LOW); + if (IS_ERR(priv->gpiod_cle)) { + err = PTR_ERR(priv->gpiod_cle); + dev_err(&pdev->dev, "CLE GPIO request failed (%d)\n", err); + goto out_mtd; + } + + /* Initialize data port direction to a known state */ + ams_delta_dir_input(priv, true); /* Scan to find existence of the device */ - err = nand_scan(ams_delta_mtd, 1); + err = nand_scan(this, 1); if (err) goto out_mtd; /* Register the partitions */ - mtd_device_register(ams_delta_mtd, partition_info, - ARRAY_SIZE(partition_info)); + mtd_device_register(mtd, partition_info, ARRAY_SIZE(partition_info)); goto out; out_mtd: - gpio_free_array(_mandatory_gpio, ARRAY_SIZE(_mandatory_gpio)); -out_gpio: - gpio_free(AMS_DELTA_GPIO_PIN_NAND_RB); iounmap(io_base); out_free: - kfree(this); out: return err; } @@ -261,18 +303,15 @@ out_free: */ static int ams_delta_cleanup(struct platform_device *pdev) { - void __iomem *io_base = platform_get_drvdata(pdev); + struct ams_delta_nand *priv = platform_get_drvdata(pdev); + struct mtd_info *mtd = nand_to_mtd(&priv->nand_chip); + void __iomem *io_base = priv->io_base; /* Release resources, unregister device */ - nand_release(ams_delta_mtd); + nand_release(mtd_to_nand(mtd)); - gpio_free_array(_mandatory_gpio, ARRAY_SIZE(_mandatory_gpio)); - gpio_free(AMS_DELTA_GPIO_PIN_NAND_RB); iounmap(io_base); - /* Free the MTD device structure */ - kfree(mtd_to_nand(ams_delta_mtd)); - return 0; } diff --git a/drivers/mtd/nand/raw/atmel/nand-controller.c b/drivers/mtd/nand/raw/atmel/nand-controller.c index a068b214ebaa..fb33f6be7c4f 100644 --- a/drivers/mtd/nand/raw/atmel/nand-controller.c +++ b/drivers/mtd/nand/raw/atmel/nand-controller.c @@ -410,25 +410,15 @@ err: return -EIO; } -static u8 atmel_nand_read_byte(struct mtd_info *mtd) +static u8 atmel_nand_read_byte(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct atmel_nand *nand = to_atmel_nand(chip); return ioread8(nand->activecs->io.virt); } -static u16 atmel_nand_read_word(struct mtd_info *mtd) +static void atmel_nand_write_byte(struct nand_chip *chip, u8 byte) { - struct nand_chip *chip = mtd_to_nand(mtd); - struct atmel_nand *nand = to_atmel_nand(chip); - - return ioread16(nand->activecs->io.virt); -} - -static void atmel_nand_write_byte(struct mtd_info *mtd, u8 byte) -{ - struct nand_chip *chip = mtd_to_nand(mtd); struct atmel_nand *nand = to_atmel_nand(chip); if (chip->options & NAND_BUSWIDTH_16) @@ -437,9 +427,8 @@ static void atmel_nand_write_byte(struct mtd_info *mtd, u8 byte) iowrite8(byte, nand->activecs->io.virt); } -static void atmel_nand_read_buf(struct mtd_info *mtd, u8 *buf, int len) +static void atmel_nand_read_buf(struct nand_chip *chip, u8 *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct atmel_nand *nand = to_atmel_nand(chip); struct atmel_nand_controller *nc; @@ -462,9 +451,8 @@ static void atmel_nand_read_buf(struct mtd_info *mtd, u8 *buf, int len) ioread8_rep(nand->activecs->io.virt, buf, len); } -static void atmel_nand_write_buf(struct mtd_info *mtd, const u8 *buf, int len) +static void atmel_nand_write_buf(struct nand_chip *chip, const u8 *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct atmel_nand *nand = to_atmel_nand(chip); struct atmel_nand_controller *nc; @@ -487,34 +475,31 @@ static void atmel_nand_write_buf(struct mtd_info *mtd, const u8 *buf, int len) iowrite8_rep(nand->activecs->io.virt, buf, len); } -static int atmel_nand_dev_ready(struct mtd_info *mtd) +static int atmel_nand_dev_ready(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct atmel_nand *nand = to_atmel_nand(chip); return gpiod_get_value(nand->activecs->rb.gpio); } -static void atmel_nand_select_chip(struct mtd_info *mtd, int cs) +static void atmel_nand_select_chip(struct nand_chip *chip, int cs) { - struct nand_chip *chip = mtd_to_nand(mtd); struct atmel_nand *nand = to_atmel_nand(chip); if (cs < 0 || cs >= nand->numcs) { nand->activecs = NULL; - chip->dev_ready = NULL; + chip->legacy.dev_ready = NULL; return; } nand->activecs = &nand->cs[cs]; if (nand->activecs->rb.type == ATMEL_NAND_GPIO_RB) - chip->dev_ready = atmel_nand_dev_ready; + chip->legacy.dev_ready = atmel_nand_dev_ready; } -static int atmel_hsmc_nand_dev_ready(struct mtd_info *mtd) +static int atmel_hsmc_nand_dev_ready(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct atmel_nand *nand = to_atmel_nand(chip); struct atmel_hsmc_nand_controller *nc; u32 status; @@ -526,15 +511,15 @@ static int atmel_hsmc_nand_dev_ready(struct mtd_info *mtd) return status & ATMEL_HSMC_NFC_SR_RBEDGE(nand->activecs->rb.id); } -static void atmel_hsmc_nand_select_chip(struct mtd_info *mtd, int cs) +static void atmel_hsmc_nand_select_chip(struct nand_chip *chip, int cs) { - struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(chip); struct atmel_nand *nand = to_atmel_nand(chip); struct atmel_hsmc_nand_controller *nc; nc = to_hsmc_nand_controller(chip->controller); - atmel_nand_select_chip(mtd, cs); + atmel_nand_select_chip(chip, cs); if (!nand->activecs) { regmap_write(nc->base.smc, ATMEL_HSMC_NFC_CTRL, @@ -543,7 +528,7 @@ static void atmel_hsmc_nand_select_chip(struct mtd_info *mtd, int cs) } if (nand->activecs->rb.type == ATMEL_NAND_NATIVE_RB) - chip->dev_ready = atmel_hsmc_nand_dev_ready; + chip->legacy.dev_ready = atmel_hsmc_nand_dev_ready; regmap_update_bits(nc->base.smc, ATMEL_HSMC_NFC_CFG, ATMEL_HSMC_NFC_CFG_PAGESIZE_MASK | @@ -607,10 +592,9 @@ static int atmel_nfc_exec_op(struct atmel_hsmc_nand_controller *nc, bool poll) return ret; } -static void atmel_hsmc_nand_cmd_ctrl(struct mtd_info *mtd, int dat, +static void atmel_hsmc_nand_cmd_ctrl(struct nand_chip *chip, int dat, unsigned int ctrl) { - struct nand_chip *chip = mtd_to_nand(mtd); struct atmel_nand *nand = to_atmel_nand(chip); struct atmel_hsmc_nand_controller *nc; @@ -634,10 +618,9 @@ static void atmel_hsmc_nand_cmd_ctrl(struct mtd_info *mtd, int dat, } } -static void atmel_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, +static void atmel_nand_cmd_ctrl(struct nand_chip *chip, int cmd, unsigned int ctrl) { - struct nand_chip *chip = mtd_to_nand(mtd); struct atmel_nand *nand = to_atmel_nand(chip); struct atmel_nand_controller *nc; @@ -851,7 +834,7 @@ static int atmel_nand_pmecc_write_pg(struct nand_chip *chip, const u8 *buf, if (ret) return ret; - atmel_nand_write_buf(mtd, buf, mtd->writesize); + atmel_nand_write_buf(chip, buf, mtd->writesize); ret = atmel_nand_pmecc_generate_eccbytes(chip, raw); if (ret) { @@ -861,20 +844,18 @@ static int atmel_nand_pmecc_write_pg(struct nand_chip *chip, const u8 *buf, atmel_nand_pmecc_disable(chip, raw); - atmel_nand_write_buf(mtd, chip->oob_poi, mtd->oobsize); + atmel_nand_write_buf(chip, chip->oob_poi, mtd->oobsize); return nand_prog_page_end_op(chip); } -static int atmel_nand_pmecc_write_page(struct mtd_info *mtd, - struct nand_chip *chip, const u8 *buf, +static int atmel_nand_pmecc_write_page(struct nand_chip *chip, const u8 *buf, int oob_required, int page) { return atmel_nand_pmecc_write_pg(chip, buf, oob_required, page, false); } -static int atmel_nand_pmecc_write_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, +static int atmel_nand_pmecc_write_page_raw(struct nand_chip *chip, const u8 *buf, int oob_required, int page) { @@ -893,8 +874,8 @@ static int atmel_nand_pmecc_read_pg(struct nand_chip *chip, u8 *buf, if (ret) return ret; - atmel_nand_read_buf(mtd, buf, mtd->writesize); - atmel_nand_read_buf(mtd, chip->oob_poi, mtd->oobsize); + atmel_nand_read_buf(chip, buf, mtd->writesize); + atmel_nand_read_buf(chip, chip->oob_poi, mtd->oobsize); ret = atmel_nand_pmecc_correct_data(chip, buf, raw); @@ -903,15 +884,13 @@ static int atmel_nand_pmecc_read_pg(struct nand_chip *chip, u8 *buf, return ret; } -static int atmel_nand_pmecc_read_page(struct mtd_info *mtd, - struct nand_chip *chip, u8 *buf, +static int atmel_nand_pmecc_read_page(struct nand_chip *chip, u8 *buf, int oob_required, int page) { return atmel_nand_pmecc_read_pg(chip, buf, oob_required, page, false); } -static int atmel_nand_pmecc_read_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, u8 *buf, +static int atmel_nand_pmecc_read_page_raw(struct nand_chip *chip, u8 *buf, int oob_required, int page) { return atmel_nand_pmecc_read_pg(chip, buf, oob_required, page, true); @@ -956,7 +935,7 @@ static int atmel_hsmc_nand_pmecc_write_pg(struct nand_chip *chip, if (ret) return ret; - atmel_nand_write_buf(mtd, chip->oob_poi, mtd->oobsize); + atmel_nand_write_buf(chip, chip->oob_poi, mtd->oobsize); nc->op.cmds[0] = NAND_CMD_PAGEPROG; nc->op.ncmds = 1; @@ -966,15 +945,14 @@ static int atmel_hsmc_nand_pmecc_write_pg(struct nand_chip *chip, dev_err(nc->base.dev, "Failed to program NAND page (err = %d)\n", ret); - status = chip->waitfunc(mtd, chip); + status = chip->legacy.waitfunc(chip); if (status & NAND_STATUS_FAIL) return -EIO; return ret; } -static int atmel_hsmc_nand_pmecc_write_page(struct mtd_info *mtd, - struct nand_chip *chip, +static int atmel_hsmc_nand_pmecc_write_page(struct nand_chip *chip, const u8 *buf, int oob_required, int page) { @@ -982,8 +960,7 @@ static int atmel_hsmc_nand_pmecc_write_page(struct mtd_info *mtd, false); } -static int atmel_hsmc_nand_pmecc_write_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, +static int atmel_hsmc_nand_pmecc_write_page_raw(struct nand_chip *chip, const u8 *buf, int oob_required, int page) { @@ -1045,16 +1022,14 @@ static int atmel_hsmc_nand_pmecc_read_pg(struct nand_chip *chip, u8 *buf, return ret; } -static int atmel_hsmc_nand_pmecc_read_page(struct mtd_info *mtd, - struct nand_chip *chip, u8 *buf, +static int atmel_hsmc_nand_pmecc_read_page(struct nand_chip *chip, u8 *buf, int oob_required, int page) { return atmel_hsmc_nand_pmecc_read_pg(chip, buf, oob_required, page, false); } -static int atmel_hsmc_nand_pmecc_read_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, +static int atmel_hsmc_nand_pmecc_read_page_raw(struct nand_chip *chip, u8 *buf, int oob_required, int page) { @@ -1473,10 +1448,9 @@ static int atmel_hsmc_nand_setup_data_interface(struct atmel_nand *nand, return 0; } -static int atmel_nand_setup_data_interface(struct mtd_info *mtd, int csline, +static int atmel_nand_setup_data_interface(struct nand_chip *chip, int csline, const struct nand_data_interface *conf) { - struct nand_chip *chip = mtd_to_nand(mtd); struct atmel_nand *nand = to_atmel_nand(chip); struct atmel_nand_controller *nc; @@ -1498,19 +1472,18 @@ static void atmel_nand_init(struct atmel_nand_controller *nc, mtd->dev.parent = nc->dev; nand->base.controller = &nc->base; - chip->cmd_ctrl = atmel_nand_cmd_ctrl; - chip->read_byte = atmel_nand_read_byte; - chip->read_word = atmel_nand_read_word; - chip->write_byte = atmel_nand_write_byte; - chip->read_buf = atmel_nand_read_buf; - chip->write_buf = atmel_nand_write_buf; + chip->legacy.cmd_ctrl = atmel_nand_cmd_ctrl; + chip->legacy.read_byte = atmel_nand_read_byte; + chip->legacy.write_byte = atmel_nand_write_byte; + chip->legacy.read_buf = atmel_nand_read_buf; + chip->legacy.write_buf = atmel_nand_write_buf; chip->select_chip = atmel_nand_select_chip; if (nc->mck && nc->caps->ops->setup_data_interface) chip->setup_data_interface = atmel_nand_setup_data_interface; /* Some NANDs require a longer delay than the default one (20us). */ - chip->chip_delay = 40; + chip->legacy.chip_delay = 40; /* * Use a bounce buffer when the buffer passed by the MTD user is not @@ -1551,7 +1524,7 @@ static void atmel_hsmc_nand_init(struct atmel_nand_controller *nc, atmel_nand_init(nc, nand); /* Overload some methods for the HSMC controller. */ - chip->cmd_ctrl = atmel_hsmc_nand_cmd_ctrl; + chip->legacy.cmd_ctrl = atmel_hsmc_nand_cmd_ctrl; chip->select_chip = atmel_hsmc_nand_select_chip; } @@ -1586,9 +1559,7 @@ static struct atmel_nand *atmel_nand_create(struct atmel_nand_controller *nc, return ERR_PTR(-EINVAL); } - nand = devm_kzalloc(nc->dev, - sizeof(*nand) + (numcs * sizeof(*nand->cs)), - GFP_KERNEL); + nand = devm_kzalloc(nc->dev, struct_size(nand, cs, numcs), GFP_KERNEL); if (!nand) { dev_err(nc->dev, "Failed to allocate NAND object\n"); return ERR_PTR(-ENOMEM); @@ -1694,7 +1665,7 @@ atmel_nand_controller_add_nand(struct atmel_nand_controller *nc, nc->caps->ops->nand_init(nc, nand); - ret = nand_scan(mtd, nand->numcs); + ret = nand_scan(chip, nand->numcs); if (ret) { dev_err(nc->dev, "NAND scan failed: %d\n", ret); return ret; @@ -2063,6 +2034,10 @@ atmel_hsmc_nand_controller_legacy_init(struct atmel_hsmc_nand_controller *nc) nand_np = dev->of_node; nfc_np = of_find_compatible_node(dev->of_node, NULL, "atmel,sama5d3-nfc"); + if (!nfc_np) { + dev_err(dev, "Could not find device node for sama5d3-nfc\n"); + return -ENODEV; + } nc->clk = of_clk_get(nfc_np, 0); if (IS_ERR(nc->clk)) { diff --git a/drivers/mtd/nand/raw/au1550nd.c b/drivers/mtd/nand/raw/au1550nd.c index 35f5c84cd331..9731c1c487f6 100644 --- a/drivers/mtd/nand/raw/au1550nd.c +++ b/drivers/mtd/nand/raw/au1550nd.c @@ -24,134 +24,113 @@ struct au1550nd_ctx { int cs; void __iomem *base; - void (*write_byte)(struct mtd_info *, u_char); + void (*write_byte)(struct nand_chip *, u_char); }; /** * au_read_byte - read one byte from the chip - * @mtd: MTD device structure + * @this: NAND chip object * * read function for 8bit buswidth */ -static u_char au_read_byte(struct mtd_info *mtd) +static u_char au_read_byte(struct nand_chip *this) { - struct nand_chip *this = mtd_to_nand(mtd); - u_char ret = readb(this->IO_ADDR_R); + u_char ret = readb(this->legacy.IO_ADDR_R); wmb(); /* drain writebuffer */ return ret; } /** * au_write_byte - write one byte to the chip - * @mtd: MTD device structure + * @this: NAND chip object * @byte: pointer to data byte to write * * write function for 8it buswidth */ -static void au_write_byte(struct mtd_info *mtd, u_char byte) +static void au_write_byte(struct nand_chip *this, u_char byte) { - struct nand_chip *this = mtd_to_nand(mtd); - writeb(byte, this->IO_ADDR_W); + writeb(byte, this->legacy.IO_ADDR_W); wmb(); /* drain writebuffer */ } /** * au_read_byte16 - read one byte endianness aware from the chip - * @mtd: MTD device structure + * @this: NAND chip object * * read function for 16bit buswidth with endianness conversion */ -static u_char au_read_byte16(struct mtd_info *mtd) +static u_char au_read_byte16(struct nand_chip *this) { - struct nand_chip *this = mtd_to_nand(mtd); - u_char ret = (u_char) cpu_to_le16(readw(this->IO_ADDR_R)); + u_char ret = (u_char) cpu_to_le16(readw(this->legacy.IO_ADDR_R)); wmb(); /* drain writebuffer */ return ret; } /** * au_write_byte16 - write one byte endianness aware to the chip - * @mtd: MTD device structure + * @this: NAND chip object * @byte: pointer to data byte to write * * write function for 16bit buswidth with endianness conversion */ -static void au_write_byte16(struct mtd_info *mtd, u_char byte) +static void au_write_byte16(struct nand_chip *this, u_char byte) { - struct nand_chip *this = mtd_to_nand(mtd); - writew(le16_to_cpu((u16) byte), this->IO_ADDR_W); + writew(le16_to_cpu((u16) byte), this->legacy.IO_ADDR_W); wmb(); /* drain writebuffer */ } /** - * au_read_word - read one word from the chip - * @mtd: MTD device structure - * - * read function for 16bit buswidth without endianness conversion - */ -static u16 au_read_word(struct mtd_info *mtd) -{ - struct nand_chip *this = mtd_to_nand(mtd); - u16 ret = readw(this->IO_ADDR_R); - wmb(); /* drain writebuffer */ - return ret; -} - -/** * au_write_buf - write buffer to chip - * @mtd: MTD device structure + * @this: NAND chip object * @buf: data buffer * @len: number of bytes to write * * write function for 8bit buswidth */ -static void au_write_buf(struct mtd_info *mtd, const u_char *buf, int len) +static void au_write_buf(struct nand_chip *this, const u_char *buf, int len) { int i; - struct nand_chip *this = mtd_to_nand(mtd); for (i = 0; i < len; i++) { - writeb(buf[i], this->IO_ADDR_W); + writeb(buf[i], this->legacy.IO_ADDR_W); wmb(); /* drain writebuffer */ } } /** * au_read_buf - read chip data into buffer - * @mtd: MTD device structure + * @this: NAND chip object * @buf: buffer to store date * @len: number of bytes to read * * read function for 8bit buswidth */ -static void au_read_buf(struct mtd_info *mtd, u_char *buf, int len) +static void au_read_buf(struct nand_chip *this, u_char *buf, int len) { int i; - struct nand_chip *this = mtd_to_nand(mtd); for (i = 0; i < len; i++) { - buf[i] = readb(this->IO_ADDR_R); + buf[i] = readb(this->legacy.IO_ADDR_R); wmb(); /* drain writebuffer */ } } /** * au_write_buf16 - write buffer to chip - * @mtd: MTD device structure + * @this: NAND chip object * @buf: data buffer * @len: number of bytes to write * * write function for 16bit buswidth */ -static void au_write_buf16(struct mtd_info *mtd, const u_char *buf, int len) +static void au_write_buf16(struct nand_chip *this, const u_char *buf, int len) { int i; - struct nand_chip *this = mtd_to_nand(mtd); u16 *p = (u16 *) buf; len >>= 1; for (i = 0; i < len; i++) { - writew(p[i], this->IO_ADDR_W); + writew(p[i], this->legacy.IO_ADDR_W); wmb(); /* drain writebuffer */ } @@ -173,7 +152,7 @@ static void au_read_buf16(struct mtd_info *mtd, u_char *buf, int len) len >>= 1; for (i = 0; i < len; i++) { - p[i] = readw(this->IO_ADDR_R); + p[i] = readw(this->legacy.IO_ADDR_R); wmb(); /* drain writebuffer */ } } @@ -200,19 +179,19 @@ static void au1550_hwcontrol(struct mtd_info *mtd, int cmd) switch (cmd) { case NAND_CTL_SETCLE: - this->IO_ADDR_W = ctx->base + MEM_STNAND_CMD; + this->legacy.IO_ADDR_W = ctx->base + MEM_STNAND_CMD; break; case NAND_CTL_CLRCLE: - this->IO_ADDR_W = ctx->base + MEM_STNAND_DATA; + this->legacy.IO_ADDR_W = ctx->base + MEM_STNAND_DATA; break; case NAND_CTL_SETALE: - this->IO_ADDR_W = ctx->base + MEM_STNAND_ADDR; + this->legacy.IO_ADDR_W = ctx->base + MEM_STNAND_ADDR; break; case NAND_CTL_CLRALE: - this->IO_ADDR_W = ctx->base + MEM_STNAND_DATA; + this->legacy.IO_ADDR_W = ctx->base + MEM_STNAND_DATA; /* FIXME: Nobody knows why this is necessary, * but it works only that way */ udelay(1); @@ -229,12 +208,12 @@ static void au1550_hwcontrol(struct mtd_info *mtd, int cmd) break; } - this->IO_ADDR_R = this->IO_ADDR_W; + this->legacy.IO_ADDR_R = this->legacy.IO_ADDR_W; wmb(); /* Drain the writebuffer */ } -int au1550_device_ready(struct mtd_info *mtd) +int au1550_device_ready(struct nand_chip *this) { return (alchemy_rdsmem(AU1000_MEM_STSTAT) & 0x1) ? 1 : 0; } @@ -248,23 +227,24 @@ int au1550_device_ready(struct mtd_info *mtd) * chip needs it to be asserted during chip not ready time but the NAND * controller keeps it released. * - * @mtd: MTD device structure + * @this: NAND chip object * @chip: chipnumber to select, -1 for deselect */ -static void au1550_select_chip(struct mtd_info *mtd, int chip) +static void au1550_select_chip(struct nand_chip *this, int chip) { } /** * au1550_command - Send command to NAND device - * @mtd: MTD device structure + * @this: NAND chip object * @command: the command to be sent * @column: the column address for this command, -1 if none * @page_addr: the page address for this command, -1 if none */ -static void au1550_command(struct mtd_info *mtd, unsigned command, int column, int page_addr) +static void au1550_command(struct nand_chip *this, unsigned command, + int column, int page_addr) { - struct nand_chip *this = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(this); struct au1550nd_ctx *ctx = container_of(this, struct au1550nd_ctx, chip); int ce_override = 0, i; @@ -289,9 +269,9 @@ static void au1550_command(struct mtd_info *mtd, unsigned command, int column, i column -= 256; readcmd = NAND_CMD_READ1; } - ctx->write_byte(mtd, readcmd); + ctx->write_byte(this, readcmd); } - ctx->write_byte(mtd, command); + ctx->write_byte(this, command); /* Set ALE and clear CLE to start address cycle */ au1550_hwcontrol(mtd, NAND_CTL_CLRCLE); @@ -305,10 +285,10 @@ static void au1550_command(struct mtd_info *mtd, unsigned command, int column, i if (this->options & NAND_BUSWIDTH_16 && !nand_opcode_8bits(command)) column >>= 1; - ctx->write_byte(mtd, column); + ctx->write_byte(this, column); } if (page_addr != -1) { - ctx->write_byte(mtd, (u8)(page_addr & 0xff)); + ctx->write_byte(this, (u8)(page_addr & 0xff)); if (command == NAND_CMD_READ0 || command == NAND_CMD_READ1 || @@ -326,10 +306,10 @@ static void au1550_command(struct mtd_info *mtd, unsigned command, int column, i au1550_hwcontrol(mtd, NAND_CTL_SETNCE); } - ctx->write_byte(mtd, (u8)(page_addr >> 8)); + ctx->write_byte(this, (u8)(page_addr >> 8)); if (this->options & NAND_ROW_ADDR_3) - ctx->write_byte(mtd, + ctx->write_byte(this, ((page_addr >> 16) & 0x0f)); } /* Latch in address */ @@ -362,7 +342,8 @@ static void au1550_command(struct mtd_info *mtd, unsigned command, int column, i /* Apply a short delay always to ensure that we do wait tWB. */ ndelay(100); /* Wait for a chip to become ready... */ - for (i = this->chip_delay; !this->dev_ready(mtd) && i > 0; --i) + for (i = this->legacy.chip_delay; + !this->legacy.dev_ready(this) && i > 0; --i) udelay(1); /* Release -CE and re-enable interrupts. */ @@ -373,7 +354,7 @@ static void au1550_command(struct mtd_info *mtd, unsigned command, int column, i /* Apply this short delay always to ensure that we do wait tWB. */ ndelay(100); - while(!this->dev_ready(mtd)); + while(!this->legacy.dev_ready(this)); } static int find_nand_cs(unsigned long nand_base) @@ -448,25 +429,24 @@ static int au1550nd_probe(struct platform_device *pdev) } ctx->cs = cs; - this->dev_ready = au1550_device_ready; + this->legacy.dev_ready = au1550_device_ready; this->select_chip = au1550_select_chip; - this->cmdfunc = au1550_command; + this->legacy.cmdfunc = au1550_command; /* 30 us command delay time */ - this->chip_delay = 30; + this->legacy.chip_delay = 30; this->ecc.mode = NAND_ECC_SOFT; this->ecc.algo = NAND_ECC_HAMMING; if (pd->devwidth) this->options |= NAND_BUSWIDTH_16; - this->read_byte = (pd->devwidth) ? au_read_byte16 : au_read_byte; + this->legacy.read_byte = (pd->devwidth) ? au_read_byte16 : au_read_byte; ctx->write_byte = (pd->devwidth) ? au_write_byte16 : au_write_byte; - this->read_word = au_read_word; - this->write_buf = (pd->devwidth) ? au_write_buf16 : au_write_buf; - this->read_buf = (pd->devwidth) ? au_read_buf16 : au_read_buf; + this->legacy.write_buf = (pd->devwidth) ? au_write_buf16 : au_write_buf; + this->legacy.read_buf = (pd->devwidth) ? au_read_buf16 : au_read_buf; - ret = nand_scan(mtd, 1); + ret = nand_scan(this, 1); if (ret) { dev_err(&pdev->dev, "NAND scan failed with %d\n", ret); goto out3; @@ -492,7 +472,7 @@ static int au1550nd_remove(struct platform_device *pdev) struct au1550nd_ctx *ctx = platform_get_drvdata(pdev); struct resource *r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - nand_release(nand_to_mtd(&ctx->chip)); + nand_release(&ctx->chip); iounmap(ctx->base); release_mem_region(r->start, 0x1000); kfree(ctx); diff --git a/drivers/mtd/nand/raw/bcm47xxnflash/main.c b/drivers/mtd/nand/raw/bcm47xxnflash/main.c index fb31429b70a9..d79694160845 100644 --- a/drivers/mtd/nand/raw/bcm47xxnflash/main.c +++ b/drivers/mtd/nand/raw/bcm47xxnflash/main.c @@ -65,7 +65,7 @@ static int bcm47xxnflash_remove(struct platform_device *pdev) { struct bcm47xxnflash *nflash = platform_get_drvdata(pdev); - nand_release(nand_to_mtd(&nflash->nand_chip)); + nand_release(&nflash->nand_chip); return 0; } diff --git a/drivers/mtd/nand/raw/bcm47xxnflash/ops_bcm4706.c b/drivers/mtd/nand/raw/bcm47xxnflash/ops_bcm4706.c index 60874de430eb..9095a79ebc7d 100644 --- a/drivers/mtd/nand/raw/bcm47xxnflash/ops_bcm4706.c +++ b/drivers/mtd/nand/raw/bcm47xxnflash/ops_bcm4706.c @@ -170,10 +170,9 @@ static void bcm47xxnflash_ops_bcm4706_write(struct mtd_info *mtd, * NAND chip ops **************************************************/ -static void bcm47xxnflash_ops_bcm4706_cmd_ctrl(struct mtd_info *mtd, int cmd, - unsigned int ctrl) +static void bcm47xxnflash_ops_bcm4706_cmd_ctrl(struct nand_chip *nand_chip, + int cmd, unsigned int ctrl) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); struct bcm47xxnflash *b47n = nand_get_controller_data(nand_chip); u32 code = 0; @@ -191,15 +190,14 @@ static void bcm47xxnflash_ops_bcm4706_cmd_ctrl(struct mtd_info *mtd, int cmd, } /* Default nand_select_chip calls cmd_ctrl, which is not used in BCM4706 */ -static void bcm47xxnflash_ops_bcm4706_select_chip(struct mtd_info *mtd, - int chip) +static void bcm47xxnflash_ops_bcm4706_select_chip(struct nand_chip *chip, + int cs) { return; } -static int bcm47xxnflash_ops_bcm4706_dev_ready(struct mtd_info *mtd) +static int bcm47xxnflash_ops_bcm4706_dev_ready(struct nand_chip *nand_chip) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); struct bcm47xxnflash *b47n = nand_get_controller_data(nand_chip); return !!(bcma_cc_read32(b47n->cc, BCMA_CC_NFLASH_CTL) & NCTL_READY); @@ -212,11 +210,11 @@ static int bcm47xxnflash_ops_bcm4706_dev_ready(struct mtd_info *mtd) * registers of ChipCommon core. Hacking cmd_ctrl to understand and convert * standard commands would be much more complicated. */ -static void bcm47xxnflash_ops_bcm4706_cmdfunc(struct mtd_info *mtd, +static void bcm47xxnflash_ops_bcm4706_cmdfunc(struct nand_chip *nand_chip, unsigned command, int column, int page_addr) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(nand_chip); struct bcm47xxnflash *b47n = nand_get_controller_data(nand_chip); struct bcma_drv_cc *cc = b47n->cc; u32 ctlcode; @@ -229,10 +227,10 @@ static void bcm47xxnflash_ops_bcm4706_cmdfunc(struct mtd_info *mtd, switch (command) { case NAND_CMD_RESET: - nand_chip->cmd_ctrl(mtd, command, NAND_CTRL_CLE); + nand_chip->legacy.cmd_ctrl(nand_chip, command, NAND_CTRL_CLE); ndelay(100); - nand_wait_ready(mtd); + nand_wait_ready(nand_chip); break; case NAND_CMD_READID: ctlcode = NCTL_CSA | 0x01000000 | NCTL_CMD1W | NCTL_CMD0; @@ -310,9 +308,9 @@ static void bcm47xxnflash_ops_bcm4706_cmdfunc(struct mtd_info *mtd, b47n->curr_command = command; } -static u8 bcm47xxnflash_ops_bcm4706_read_byte(struct mtd_info *mtd) +static u8 bcm47xxnflash_ops_bcm4706_read_byte(struct nand_chip *nand_chip) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(nand_chip); struct bcm47xxnflash *b47n = nand_get_controller_data(nand_chip); struct bcma_drv_cc *cc = b47n->cc; u32 tmp = 0; @@ -338,31 +336,31 @@ static u8 bcm47xxnflash_ops_bcm4706_read_byte(struct mtd_info *mtd) return 0; } -static void bcm47xxnflash_ops_bcm4706_read_buf(struct mtd_info *mtd, +static void bcm47xxnflash_ops_bcm4706_read_buf(struct nand_chip *nand_chip, uint8_t *buf, int len) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); struct bcm47xxnflash *b47n = nand_get_controller_data(nand_chip); switch (b47n->curr_command) { case NAND_CMD_READ0: case NAND_CMD_READOOB: - bcm47xxnflash_ops_bcm4706_read(mtd, buf, len); + bcm47xxnflash_ops_bcm4706_read(nand_to_mtd(nand_chip), buf, + len); return; } pr_err("Invalid command for buf read: 0x%X\n", b47n->curr_command); } -static void bcm47xxnflash_ops_bcm4706_write_buf(struct mtd_info *mtd, +static void bcm47xxnflash_ops_bcm4706_write_buf(struct nand_chip *nand_chip, const uint8_t *buf, int len) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); struct bcm47xxnflash *b47n = nand_get_controller_data(nand_chip); switch (b47n->curr_command) { case NAND_CMD_SEQIN: - bcm47xxnflash_ops_bcm4706_write(mtd, buf, len); + bcm47xxnflash_ops_bcm4706_write(nand_to_mtd(nand_chip), buf, + len); return; } @@ -386,16 +384,16 @@ int bcm47xxnflash_ops_bcm4706_init(struct bcm47xxnflash *b47n) u32 val; b47n->nand_chip.select_chip = bcm47xxnflash_ops_bcm4706_select_chip; - nand_chip->cmd_ctrl = bcm47xxnflash_ops_bcm4706_cmd_ctrl; - nand_chip->dev_ready = bcm47xxnflash_ops_bcm4706_dev_ready; - b47n->nand_chip.cmdfunc = bcm47xxnflash_ops_bcm4706_cmdfunc; - b47n->nand_chip.read_byte = bcm47xxnflash_ops_bcm4706_read_byte; - b47n->nand_chip.read_buf = bcm47xxnflash_ops_bcm4706_read_buf; - b47n->nand_chip.write_buf = bcm47xxnflash_ops_bcm4706_write_buf; - b47n->nand_chip.set_features = nand_get_set_features_notsupp; - b47n->nand_chip.get_features = nand_get_set_features_notsupp; - - nand_chip->chip_delay = 50; + nand_chip->legacy.cmd_ctrl = bcm47xxnflash_ops_bcm4706_cmd_ctrl; + nand_chip->legacy.dev_ready = bcm47xxnflash_ops_bcm4706_dev_ready; + b47n->nand_chip.legacy.cmdfunc = bcm47xxnflash_ops_bcm4706_cmdfunc; + b47n->nand_chip.legacy.read_byte = bcm47xxnflash_ops_bcm4706_read_byte; + b47n->nand_chip.legacy.read_buf = bcm47xxnflash_ops_bcm4706_read_buf; + b47n->nand_chip.legacy.write_buf = bcm47xxnflash_ops_bcm4706_write_buf; + b47n->nand_chip.legacy.set_features = nand_get_set_features_notsupp; + b47n->nand_chip.legacy.get_features = nand_get_set_features_notsupp; + + nand_chip->legacy.chip_delay = 50; b47n->nand_chip.bbt_options = NAND_BBT_USE_FLASH; b47n->nand_chip.ecc.mode = NAND_ECC_NONE; /* TODO: implement ECC */ @@ -423,7 +421,7 @@ int bcm47xxnflash_ops_bcm4706_init(struct bcm47xxnflash *b47n) (w4 << 24 | w3 << 18 | w2 << 12 | w1 << 6 | w0)); /* Scan NAND */ - err = nand_scan(nand_to_mtd(&b47n->nand_chip), 1); + err = nand_scan(&b47n->nand_chip, 1); if (err) { pr_err("Could not scan NAND flash: %d\n", err); goto exit; diff --git a/drivers/mtd/nand/raw/brcmnand/brcmnand.c b/drivers/mtd/nand/raw/brcmnand/brcmnand.c index 4b90d5b380c2..482c6f093f99 100644 --- a/drivers/mtd/nand/raw/brcmnand/brcmnand.c +++ b/drivers/mtd/nand/raw/brcmnand/brcmnand.c @@ -1231,15 +1231,14 @@ static void brcmnand_send_cmd(struct brcmnand_host *host, int cmd) * NAND MTD API: read/program/erase ***********************************************************************/ -static void brcmnand_cmd_ctrl(struct mtd_info *mtd, int dat, - unsigned int ctrl) +static void brcmnand_cmd_ctrl(struct nand_chip *chip, int dat, + unsigned int ctrl) { /* intentionally left blank */ } -static int brcmnand_waitfunc(struct mtd_info *mtd, struct nand_chip *this) +static int brcmnand_waitfunc(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct brcmnand_host *host = nand_get_controller_data(chip); struct brcmnand_controller *ctrl = host->ctrl; unsigned long timeo = msecs_to_jiffies(100); @@ -1274,7 +1273,6 @@ static int brcmnand_low_level_op(struct brcmnand_host *host, enum brcmnand_llop_type type, u32 data, bool last_op) { - struct mtd_info *mtd = nand_to_mtd(&host->chip); struct nand_chip *chip = &host->chip; struct brcmnand_controller *ctrl = host->ctrl; u32 tmp; @@ -1307,13 +1305,13 @@ static int brcmnand_low_level_op(struct brcmnand_host *host, (void)brcmnand_read_reg(ctrl, BRCMNAND_LL_OP); brcmnand_send_cmd(host, CMD_LOW_LEVEL_OP); - return brcmnand_waitfunc(mtd, chip); + return brcmnand_waitfunc(chip); } -static void brcmnand_cmdfunc(struct mtd_info *mtd, unsigned command, +static void brcmnand_cmdfunc(struct nand_chip *chip, unsigned command, int column, int page_addr) { - struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(chip); struct brcmnand_host *host = nand_get_controller_data(chip); struct brcmnand_controller *ctrl = host->ctrl; u64 addr = (u64)page_addr << chip->page_shift; @@ -1383,7 +1381,7 @@ static void brcmnand_cmdfunc(struct mtd_info *mtd, unsigned command, (void)brcmnand_read_reg(ctrl, BRCMNAND_CMD_ADDRESS); brcmnand_send_cmd(host, native_cmd); - brcmnand_waitfunc(mtd, chip); + brcmnand_waitfunc(chip); if (native_cmd == CMD_PARAMETER_READ || native_cmd == CMD_PARAMETER_CHANGE_COL) { @@ -1417,9 +1415,8 @@ static void brcmnand_cmdfunc(struct mtd_info *mtd, unsigned command, brcmnand_wp(mtd, 1); } -static uint8_t brcmnand_read_byte(struct mtd_info *mtd) +static uint8_t brcmnand_read_byte(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct brcmnand_host *host = nand_get_controller_data(chip); struct brcmnand_controller *ctrl = host->ctrl; uint8_t ret = 0; @@ -1474,19 +1471,18 @@ static uint8_t brcmnand_read_byte(struct mtd_info *mtd) return ret; } -static void brcmnand_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) +static void brcmnand_read_buf(struct nand_chip *chip, uint8_t *buf, int len) { int i; for (i = 0; i < len; i++, buf++) - *buf = brcmnand_read_byte(mtd); + *buf = brcmnand_read_byte(chip); } -static void brcmnand_write_buf(struct mtd_info *mtd, const uint8_t *buf, - int len) +static void brcmnand_write_buf(struct nand_chip *chip, const uint8_t *buf, + int len) { int i; - struct nand_chip *chip = mtd_to_nand(mtd); struct brcmnand_host *host = nand_get_controller_data(chip); switch (host->last_cmd) { @@ -1617,7 +1613,7 @@ static int brcmnand_read_by_pio(struct mtd_info *mtd, struct nand_chip *chip, (void)brcmnand_read_reg(ctrl, BRCMNAND_CMD_ADDRESS); /* SPARE_AREA_READ does not use ECC, so just use PAGE_READ */ brcmnand_send_cmd(host, CMD_PAGE_READ); - brcmnand_waitfunc(mtd, chip); + brcmnand_waitfunc(chip); if (likely(buf)) { brcmnand_soc_data_bus_prepare(ctrl->soc, false); @@ -1689,7 +1685,7 @@ static int brcmstb_nand_verify_erased_page(struct mtd_info *mtd, sas = mtd->oobsize / chip->ecc.steps; /* read without ecc for verification */ - ret = chip->ecc.read_page_raw(mtd, chip, buf, true, page); + ret = chip->ecc.read_page_raw(chip, buf, true, page); if (ret) return ret; @@ -1786,9 +1782,10 @@ try_dmaread: return 0; } -static int brcmnand_read_page(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int brcmnand_read_page(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct brcmnand_host *host = nand_get_controller_data(chip); u8 *oob = oob_required ? (u8 *)chip->oob_poi : NULL; @@ -1798,10 +1795,11 @@ static int brcmnand_read_page(struct mtd_info *mtd, struct nand_chip *chip, mtd->writesize >> FC_SHIFT, (u32 *)buf, oob); } -static int brcmnand_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int brcmnand_read_page_raw(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { struct brcmnand_host *host = nand_get_controller_data(chip); + struct mtd_info *mtd = nand_to_mtd(chip); u8 *oob = oob_required ? (u8 *)chip->oob_poi : NULL; int ret; @@ -1814,17 +1812,18 @@ static int brcmnand_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip, return ret; } -static int brcmnand_read_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int brcmnand_read_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + return brcmnand_read(mtd, chip, (u64)page << chip->page_shift, mtd->writesize >> FC_SHIFT, NULL, (u8 *)chip->oob_poi); } -static int brcmnand_read_oob_raw(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int brcmnand_read_oob_raw(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct brcmnand_host *host = nand_get_controller_data(chip); brcmnand_set_ecc_enabled(host, 0); @@ -1892,7 +1891,7 @@ static int brcmnand_write(struct mtd_info *mtd, struct nand_chip *chip, /* we cannot use SPARE_AREA_PROGRAM when PARTIAL_PAGE_EN=0 */ brcmnand_send_cmd(host, CMD_PROGRAM_PAGE); - status = brcmnand_waitfunc(mtd, chip); + status = brcmnand_waitfunc(chip); if (status & NAND_STATUS_FAIL) { dev_info(ctrl->dev, "program failed at %llx\n", @@ -1906,9 +1905,10 @@ out: return ret; } -static int brcmnand_write_page(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, int page) +static int brcmnand_write_page(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct brcmnand_host *host = nand_get_controller_data(chip); void *oob = oob_required ? chip->oob_poi : NULL; @@ -1918,10 +1918,10 @@ static int brcmnand_write_page(struct mtd_info *mtd, struct nand_chip *chip, return nand_prog_page_end_op(chip); } -static int brcmnand_write_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, const uint8_t *buf, +static int brcmnand_write_page_raw(struct nand_chip *chip, const uint8_t *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct brcmnand_host *host = nand_get_controller_data(chip); void *oob = oob_required ? chip->oob_poi : NULL; @@ -1933,16 +1933,16 @@ static int brcmnand_write_page_raw(struct mtd_info *mtd, return nand_prog_page_end_op(chip); } -static int brcmnand_write_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int brcmnand_write_oob(struct nand_chip *chip, int page) { - return brcmnand_write(mtd, chip, (u64)page << chip->page_shift, - NULL, chip->oob_poi); + return brcmnand_write(nand_to_mtd(chip), chip, + (u64)page << chip->page_shift, NULL, + chip->oob_poi); } -static int brcmnand_write_oob_raw(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int brcmnand_write_oob_raw(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct brcmnand_host *host = nand_get_controller_data(chip); int ret; @@ -2270,15 +2270,12 @@ static int brcmnand_init_cs(struct brcmnand_host *host, struct device_node *dn) mtd->owner = THIS_MODULE; mtd->dev.parent = &pdev->dev; - chip->IO_ADDR_R = (void __iomem *)0xdeadbeef; - chip->IO_ADDR_W = (void __iomem *)0xdeadbeef; - - chip->cmd_ctrl = brcmnand_cmd_ctrl; - chip->cmdfunc = brcmnand_cmdfunc; - chip->waitfunc = brcmnand_waitfunc; - chip->read_byte = brcmnand_read_byte; - chip->read_buf = brcmnand_read_buf; - chip->write_buf = brcmnand_write_buf; + chip->legacy.cmd_ctrl = brcmnand_cmd_ctrl; + chip->legacy.cmdfunc = brcmnand_cmdfunc; + chip->legacy.waitfunc = brcmnand_waitfunc; + chip->legacy.read_byte = brcmnand_read_byte; + chip->legacy.read_buf = brcmnand_read_buf; + chip->legacy.write_buf = brcmnand_write_buf; chip->ecc.mode = NAND_ECC_HW; chip->ecc.read_page = brcmnand_read_page; @@ -2301,7 +2298,7 @@ static int brcmnand_init_cs(struct brcmnand_host *host, struct device_node *dn) nand_writereg(ctrl, cfg_offs, nand_readreg(ctrl, cfg_offs) & ~CFG_BUS_WIDTH); - ret = nand_scan(mtd, 1); + ret = nand_scan(chip, 1); if (ret) return ret; @@ -2616,7 +2613,7 @@ int brcmnand_remove(struct platform_device *pdev) struct brcmnand_host *host; list_for_each_entry(host, &ctrl->host_list, node) - nand_release(nand_to_mtd(&host->chip)); + nand_release(&host->chip); clk_disable_unprepare(ctrl->clk); diff --git a/drivers/mtd/nand/raw/cafe_nand.c b/drivers/mtd/nand/raw/cafe_nand.c index 1dbe43adcfe7..c1a745940d12 100644 --- a/drivers/mtd/nand/raw/cafe_nand.c +++ b/drivers/mtd/nand/raw/cafe_nand.c @@ -100,9 +100,8 @@ static const char *part_probes[] = { "cmdlinepart", "RedBoot", NULL }; #define cafe_readl(cafe, addr) readl((cafe)->mmio + CAFE_##addr) #define cafe_writel(cafe, datum, addr) writel(datum, (cafe)->mmio + CAFE_##addr) -static int cafe_device_ready(struct mtd_info *mtd) +static int cafe_device_ready(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct cafe_priv *cafe = nand_get_controller_data(chip); int result = !!(cafe_readl(cafe, NAND_STATUS) & 0x40000000); uint32_t irqs = cafe_readl(cafe, NAND_IRQ); @@ -117,9 +116,8 @@ static int cafe_device_ready(struct mtd_info *mtd) } -static void cafe_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len) +static void cafe_write_buf(struct nand_chip *chip, const uint8_t *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct cafe_priv *cafe = nand_get_controller_data(chip); if (cafe->usedma) @@ -133,9 +131,8 @@ static void cafe_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len) len, cafe->datalen); } -static void cafe_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) +static void cafe_read_buf(struct nand_chip *chip, uint8_t *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct cafe_priv *cafe = nand_get_controller_data(chip); if (cafe->usedma) @@ -148,22 +145,21 @@ static void cafe_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) cafe->datalen += len; } -static uint8_t cafe_read_byte(struct mtd_info *mtd) +static uint8_t cafe_read_byte(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct cafe_priv *cafe = nand_get_controller_data(chip); uint8_t d; - cafe_read_buf(mtd, &d, 1); + cafe_read_buf(chip, &d, 1); cafe_dev_dbg(&cafe->pdev->dev, "Read %02x\n", d); return d; } -static void cafe_nand_cmdfunc(struct mtd_info *mtd, unsigned command, +static void cafe_nand_cmdfunc(struct nand_chip *chip, unsigned command, int column, int page_addr) { - struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(chip); struct cafe_priv *cafe = nand_get_controller_data(chip); int adrbytes = 0; uint32_t ctl1; @@ -313,13 +309,12 @@ static void cafe_nand_cmdfunc(struct mtd_info *mtd, unsigned command, cafe_writel(cafe, cafe->ctl2, NAND_CTRL2); return; } - nand_wait_ready(mtd); + nand_wait_ready(chip); cafe_writel(cafe, cafe->ctl2, NAND_CTRL2); } -static void cafe_select_chip(struct mtd_info *mtd, int chipnr) +static void cafe_select_chip(struct nand_chip *chip, int chipnr) { - struct nand_chip *chip = mtd_to_nand(mtd); struct cafe_priv *cafe = nand_get_controller_data(chip); cafe_dev_dbg(&cafe->pdev->dev, "select_chip %d\n", chipnr); @@ -346,17 +341,19 @@ static irqreturn_t cafe_nand_interrupt(int irq, void *id) return IRQ_HANDLED; } -static int cafe_nand_write_oob(struct mtd_info *mtd, - struct nand_chip *chip, int page) +static int cafe_nand_write_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + return nand_prog_page_op(chip, page, mtd->writesize, chip->oob_poi, mtd->oobsize); } /* Don't use -- use nand_read_oob_std for now */ -static int cafe_nand_read_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int cafe_nand_read_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + return nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize); } /** @@ -369,9 +366,10 @@ static int cafe_nand_read_oob(struct mtd_info *mtd, struct nand_chip *chip, * The hw generator calculates the error syndrome automatically. Therefore * we need a special oob layout and handling. */ -static int cafe_nand_read_page(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int cafe_nand_read_page(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct cafe_priv *cafe = nand_get_controller_data(chip); unsigned int max_bitflips = 0; @@ -380,7 +378,7 @@ static int cafe_nand_read_page(struct mtd_info *mtd, struct nand_chip *chip, cafe_readl(cafe, NAND_ECC_SYN01)); nand_read_page_op(chip, page, 0, buf, mtd->writesize); - chip->read_buf(mtd, chip->oob_poi, mtd->oobsize); + chip->legacy.read_buf(chip, chip->oob_poi, mtd->oobsize); if (checkecc && cafe_readl(cafe, NAND_ECC_RESULT) & (1<<18)) { unsigned short syn[8], pat[4]; @@ -531,15 +529,15 @@ static struct nand_bbt_descr cafe_bbt_mirror_descr_512 = { }; -static int cafe_nand_write_page_lowlevel(struct mtd_info *mtd, - struct nand_chip *chip, - const uint8_t *buf, int oob_required, - int page) +static int cafe_nand_write_page_lowlevel(struct nand_chip *chip, + const uint8_t *buf, int oob_required, + int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct cafe_priv *cafe = nand_get_controller_data(chip); nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize); - chip->write_buf(mtd, chip->oob_poi, mtd->oobsize); + chip->legacy.write_buf(chip, chip->oob_poi, mtd->oobsize); /* Set up ECC autogeneration */ cafe->ctl2 |= (1<<30); @@ -547,7 +545,7 @@ static int cafe_nand_write_page_lowlevel(struct mtd_info *mtd, return nand_prog_page_end_op(chip); } -static int cafe_nand_block_bad(struct mtd_info *mtd, loff_t ofs) +static int cafe_nand_block_bad(struct nand_chip *chip, loff_t ofs) { return 0; } @@ -705,23 +703,23 @@ static int cafe_nand_probe(struct pci_dev *pdev, goto out_ior; } - cafe->nand.cmdfunc = cafe_nand_cmdfunc; - cafe->nand.dev_ready = cafe_device_ready; - cafe->nand.read_byte = cafe_read_byte; - cafe->nand.read_buf = cafe_read_buf; - cafe->nand.write_buf = cafe_write_buf; + cafe->nand.legacy.cmdfunc = cafe_nand_cmdfunc; + cafe->nand.legacy.dev_ready = cafe_device_ready; + cafe->nand.legacy.read_byte = cafe_read_byte; + cafe->nand.legacy.read_buf = cafe_read_buf; + cafe->nand.legacy.write_buf = cafe_write_buf; cafe->nand.select_chip = cafe_select_chip; - cafe->nand.set_features = nand_get_set_features_notsupp; - cafe->nand.get_features = nand_get_set_features_notsupp; + cafe->nand.legacy.set_features = nand_get_set_features_notsupp; + cafe->nand.legacy.get_features = nand_get_set_features_notsupp; - cafe->nand.chip_delay = 0; + cafe->nand.legacy.chip_delay = 0; /* Enable the following for a flash based bad block table */ cafe->nand.bbt_options = NAND_BBT_USE_FLASH; if (skipbbt) { cafe->nand.options |= NAND_SKIP_BBTSCAN; - cafe->nand.block_bad = cafe_nand_block_bad; + cafe->nand.legacy.block_bad = cafe_nand_block_bad; } if (numtimings && numtimings != 3) { @@ -783,7 +781,7 @@ static int cafe_nand_probe(struct pci_dev *pdev, /* Scan to find existence of the device */ cafe->nand.dummy_controller.ops = &cafe_nand_controller_ops; - err = nand_scan(mtd, 2); + err = nand_scan(&cafe->nand, 2); if (err) goto out_irq; @@ -819,7 +817,7 @@ static void cafe_nand_remove(struct pci_dev *pdev) /* Disable NAND IRQ in global IRQ mask register */ cafe_writel(cafe, ~1 & cafe_readl(cafe, GLOBAL_IRQ_MASK), GLOBAL_IRQ_MASK); free_irq(pdev->irq, mtd); - nand_release(mtd); + nand_release(chip); free_rs(cafe->rs); pci_iounmap(pdev, cafe->mmio); dma_free_coherent(&cafe->pdev->dev, 2112, cafe->dmabuf, cafe->dmaaddr); diff --git a/drivers/mtd/nand/raw/cmx270_nand.c b/drivers/mtd/nand/raw/cmx270_nand.c index b66e254b6802..143e4acacaae 100644 --- a/drivers/mtd/nand/raw/cmx270_nand.c +++ b/drivers/mtd/nand/raw/cmx270_nand.c @@ -49,29 +49,26 @@ static const struct mtd_partition partition_info[] = { }; #define NUM_PARTITIONS (ARRAY_SIZE(partition_info)) -static u_char cmx270_read_byte(struct mtd_info *mtd) +static u_char cmx270_read_byte(struct nand_chip *this) { - struct nand_chip *this = mtd_to_nand(mtd); - - return (readl(this->IO_ADDR_R) >> 16); + return (readl(this->legacy.IO_ADDR_R) >> 16); } -static void cmx270_write_buf(struct mtd_info *mtd, const u_char *buf, int len) +static void cmx270_write_buf(struct nand_chip *this, const u_char *buf, + int len) { int i; - struct nand_chip *this = mtd_to_nand(mtd); for (i=0; i<len; i++) - writel((*buf++ << 16), this->IO_ADDR_W); + writel((*buf++ << 16), this->legacy.IO_ADDR_W); } -static void cmx270_read_buf(struct mtd_info *mtd, u_char *buf, int len) +static void cmx270_read_buf(struct nand_chip *this, u_char *buf, int len) { int i; - struct nand_chip *this = mtd_to_nand(mtd); for (i=0; i<len; i++) - *buf++ = readl(this->IO_ADDR_R) >> 16; + *buf++ = readl(this->legacy.IO_ADDR_R) >> 16; } static inline void nand_cs_on(void) @@ -89,11 +86,10 @@ static void nand_cs_off(void) /* * hardware specific access to control-lines */ -static void cmx270_hwcontrol(struct mtd_info *mtd, int dat, +static void cmx270_hwcontrol(struct nand_chip *this, int dat, unsigned int ctrl) { - struct nand_chip *this = mtd_to_nand(mtd); - unsigned int nandaddr = (unsigned int)this->IO_ADDR_W; + unsigned int nandaddr = (unsigned int)this->legacy.IO_ADDR_W; dsb(); @@ -113,9 +109,9 @@ static void cmx270_hwcontrol(struct mtd_info *mtd, int dat, } dsb(); - this->IO_ADDR_W = (void __iomem*)nandaddr; + this->legacy.IO_ADDR_W = (void __iomem*)nandaddr; if (dat != NAND_CMD_NONE) - writel((dat << 16), this->IO_ADDR_W); + writel((dat << 16), this->legacy.IO_ADDR_W); dsb(); } @@ -123,7 +119,7 @@ static void cmx270_hwcontrol(struct mtd_info *mtd, int dat, /* * read device ready pin */ -static int cmx270_device_ready(struct mtd_info *mtd) +static int cmx270_device_ready(struct nand_chip *this) { dsb(); @@ -177,23 +173,23 @@ static int __init cmx270_init(void) cmx270_nand_mtd->owner = THIS_MODULE; /* insert callbacks */ - this->IO_ADDR_R = cmx270_nand_io; - this->IO_ADDR_W = cmx270_nand_io; - this->cmd_ctrl = cmx270_hwcontrol; - this->dev_ready = cmx270_device_ready; + this->legacy.IO_ADDR_R = cmx270_nand_io; + this->legacy.IO_ADDR_W = cmx270_nand_io; + this->legacy.cmd_ctrl = cmx270_hwcontrol; + this->legacy.dev_ready = cmx270_device_ready; /* 15 us command delay time */ - this->chip_delay = 20; + this->legacy.chip_delay = 20; this->ecc.mode = NAND_ECC_SOFT; this->ecc.algo = NAND_ECC_HAMMING; /* read/write functions */ - this->read_byte = cmx270_read_byte; - this->read_buf = cmx270_read_buf; - this->write_buf = cmx270_write_buf; + this->legacy.read_byte = cmx270_read_byte; + this->legacy.read_buf = cmx270_read_buf; + this->legacy.write_buf = cmx270_write_buf; /* Scan to find existence of the device */ - ret = nand_scan(cmx270_nand_mtd, 1); + ret = nand_scan(this, 1); if (ret) { pr_notice("No NAND device\n"); goto err_scan; @@ -228,7 +224,7 @@ module_init(cmx270_init); static void __exit cmx270_cleanup(void) { /* Release resources, unregister device */ - nand_release(cmx270_nand_mtd); + nand_release(mtd_to_nand(cmx270_nand_mtd)); gpio_free(GPIO_NAND_RB); gpio_free(GPIO_NAND_CS); diff --git a/drivers/mtd/nand/raw/cs553x_nand.c b/drivers/mtd/nand/raw/cs553x_nand.c index beafad62e7d5..c6f578aff5d9 100644 --- a/drivers/mtd/nand/raw/cs553x_nand.c +++ b/drivers/mtd/nand/raw/cs553x_nand.c @@ -93,83 +93,74 @@ #define CS_NAND_ECC_CLRECC (1<<1) #define CS_NAND_ECC_ENECC (1<<0) -static void cs553x_read_buf(struct mtd_info *mtd, u_char *buf, int len) +static void cs553x_read_buf(struct nand_chip *this, u_char *buf, int len) { - struct nand_chip *this = mtd_to_nand(mtd); - while (unlikely(len > 0x800)) { - memcpy_fromio(buf, this->IO_ADDR_R, 0x800); + memcpy_fromio(buf, this->legacy.IO_ADDR_R, 0x800); buf += 0x800; len -= 0x800; } - memcpy_fromio(buf, this->IO_ADDR_R, len); + memcpy_fromio(buf, this->legacy.IO_ADDR_R, len); } -static void cs553x_write_buf(struct mtd_info *mtd, const u_char *buf, int len) +static void cs553x_write_buf(struct nand_chip *this, const u_char *buf, int len) { - struct nand_chip *this = mtd_to_nand(mtd); - while (unlikely(len > 0x800)) { - memcpy_toio(this->IO_ADDR_R, buf, 0x800); + memcpy_toio(this->legacy.IO_ADDR_R, buf, 0x800); buf += 0x800; len -= 0x800; } - memcpy_toio(this->IO_ADDR_R, buf, len); + memcpy_toio(this->legacy.IO_ADDR_R, buf, len); } -static unsigned char cs553x_read_byte(struct mtd_info *mtd) +static unsigned char cs553x_read_byte(struct nand_chip *this) { - struct nand_chip *this = mtd_to_nand(mtd); - return readb(this->IO_ADDR_R); + return readb(this->legacy.IO_ADDR_R); } -static void cs553x_write_byte(struct mtd_info *mtd, u_char byte) +static void cs553x_write_byte(struct nand_chip *this, u_char byte) { - struct nand_chip *this = mtd_to_nand(mtd); int i = 100000; - while (i && readb(this->IO_ADDR_R + MM_NAND_STS) & CS_NAND_CTLR_BUSY) { + while (i && readb(this->legacy.IO_ADDR_R + MM_NAND_STS) & CS_NAND_CTLR_BUSY) { udelay(1); i--; } - writeb(byte, this->IO_ADDR_W + 0x801); + writeb(byte, this->legacy.IO_ADDR_W + 0x801); } -static void cs553x_hwcontrol(struct mtd_info *mtd, int cmd, +static void cs553x_hwcontrol(struct nand_chip *this, int cmd, unsigned int ctrl) { - struct nand_chip *this = mtd_to_nand(mtd); - void __iomem *mmio_base = this->IO_ADDR_R; + void __iomem *mmio_base = this->legacy.IO_ADDR_R; if (ctrl & NAND_CTRL_CHANGE) { unsigned char ctl = (ctrl & ~NAND_CTRL_CHANGE ) ^ 0x01; writeb(ctl, mmio_base + MM_NAND_CTL); } if (cmd != NAND_CMD_NONE) - cs553x_write_byte(mtd, cmd); + cs553x_write_byte(this, cmd); } -static int cs553x_device_ready(struct mtd_info *mtd) +static int cs553x_device_ready(struct nand_chip *this) { - struct nand_chip *this = mtd_to_nand(mtd); - void __iomem *mmio_base = this->IO_ADDR_R; + void __iomem *mmio_base = this->legacy.IO_ADDR_R; unsigned char foo = readb(mmio_base + MM_NAND_STS); return (foo & CS_NAND_STS_FLASH_RDY) && !(foo & CS_NAND_CTLR_BUSY); } -static void cs_enable_hwecc(struct mtd_info *mtd, int mode) +static void cs_enable_hwecc(struct nand_chip *this, int mode) { - struct nand_chip *this = mtd_to_nand(mtd); - void __iomem *mmio_base = this->IO_ADDR_R; + void __iomem *mmio_base = this->legacy.IO_ADDR_R; writeb(0x07, mmio_base + MM_NAND_ECC_CTL); } -static int cs_calculate_ecc(struct mtd_info *mtd, const u_char *dat, u_char *ecc_code) +static int cs_calculate_ecc(struct nand_chip *this, const u_char *dat, + u_char *ecc_code) { uint32_t ecc; - struct nand_chip *this = mtd_to_nand(mtd); - void __iomem *mmio_base = this->IO_ADDR_R; + void __iomem *mmio_base = this->legacy.IO_ADDR_R; ecc = readl(mmio_base + MM_NAND_STS); @@ -208,20 +199,20 @@ static int __init cs553x_init_one(int cs, int mmio, unsigned long adr) new_mtd->owner = THIS_MODULE; /* map physical address */ - this->IO_ADDR_R = this->IO_ADDR_W = ioremap(adr, 4096); - if (!this->IO_ADDR_R) { + this->legacy.IO_ADDR_R = this->legacy.IO_ADDR_W = ioremap(adr, 4096); + if (!this->legacy.IO_ADDR_R) { pr_warn("ioremap cs553x NAND @0x%08lx failed\n", adr); err = -EIO; goto out_mtd; } - this->cmd_ctrl = cs553x_hwcontrol; - this->dev_ready = cs553x_device_ready; - this->read_byte = cs553x_read_byte; - this->read_buf = cs553x_read_buf; - this->write_buf = cs553x_write_buf; + this->legacy.cmd_ctrl = cs553x_hwcontrol; + this->legacy.dev_ready = cs553x_device_ready; + this->legacy.read_byte = cs553x_read_byte; + this->legacy.read_buf = cs553x_read_buf; + this->legacy.write_buf = cs553x_write_buf; - this->chip_delay = 0; + this->legacy.chip_delay = 0; this->ecc.mode = NAND_ECC_HW; this->ecc.size = 256; @@ -241,7 +232,7 @@ static int __init cs553x_init_one(int cs, int mmio, unsigned long adr) } /* Scan to find existence of the device */ - err = nand_scan(new_mtd, 1); + err = nand_scan(this, 1); if (err) goto out_free; @@ -251,7 +242,7 @@ static int __init cs553x_init_one(int cs, int mmio, unsigned long adr) out_free: kfree(new_mtd->name); out_ior: - iounmap(this->IO_ADDR_R); + iounmap(this->legacy.IO_ADDR_R); out_mtd: kfree(this); out: @@ -333,10 +324,10 @@ static void __exit cs553x_cleanup(void) continue; this = mtd_to_nand(mtd); - mmio_base = this->IO_ADDR_R; + mmio_base = this->legacy.IO_ADDR_R; /* Release resources, unregister device */ - nand_release(mtd); + nand_release(this); kfree(mtd->name); cs553x_mtd[i] = NULL; diff --git a/drivers/mtd/nand/raw/davinci_nand.c b/drivers/mtd/nand/raw/davinci_nand.c index 40145e206a6b..80f228d23cd2 100644 --- a/drivers/mtd/nand/raw/davinci_nand.c +++ b/drivers/mtd/nand/raw/davinci_nand.c @@ -97,12 +97,11 @@ static inline void davinci_nand_writel(struct davinci_nand_info *info, * Access to hardware control lines: ALE, CLE, secondary chipselect. */ -static void nand_davinci_hwcontrol(struct mtd_info *mtd, int cmd, +static void nand_davinci_hwcontrol(struct nand_chip *nand, int cmd, unsigned int ctrl) { - struct davinci_nand_info *info = to_davinci_nand(mtd); + struct davinci_nand_info *info = to_davinci_nand(nand_to_mtd(nand)); void __iomem *addr = info->current_cs; - struct nand_chip *nand = mtd_to_nand(mtd); /* Did the control lines change? */ if (ctrl & NAND_CTRL_CHANGE) { @@ -111,16 +110,16 @@ static void nand_davinci_hwcontrol(struct mtd_info *mtd, int cmd, else if ((ctrl & NAND_CTRL_ALE) == NAND_CTRL_ALE) addr += info->mask_ale; - nand->IO_ADDR_W = addr; + nand->legacy.IO_ADDR_W = addr; } if (cmd != NAND_CMD_NONE) - iowrite8(cmd, nand->IO_ADDR_W); + iowrite8(cmd, nand->legacy.IO_ADDR_W); } -static void nand_davinci_select_chip(struct mtd_info *mtd, int chip) +static void nand_davinci_select_chip(struct nand_chip *nand, int chip) { - struct davinci_nand_info *info = to_davinci_nand(mtd); + struct davinci_nand_info *info = to_davinci_nand(nand_to_mtd(nand)); info->current_cs = info->vaddr; @@ -128,8 +127,8 @@ static void nand_davinci_select_chip(struct mtd_info *mtd, int chip) if (chip > 0) info->current_cs += info->mask_chipsel; - info->chip.IO_ADDR_W = info->current_cs; - info->chip.IO_ADDR_R = info->chip.IO_ADDR_W; + info->chip.legacy.IO_ADDR_W = info->current_cs; + info->chip.legacy.IO_ADDR_R = info->chip.legacy.IO_ADDR_W; } /*----------------------------------------------------------------------*/ @@ -146,16 +145,16 @@ static inline uint32_t nand_davinci_readecc_1bit(struct mtd_info *mtd) + 4 * info->core_chipsel); } -static void nand_davinci_hwctl_1bit(struct mtd_info *mtd, int mode) +static void nand_davinci_hwctl_1bit(struct nand_chip *chip, int mode) { struct davinci_nand_info *info; uint32_t nandcfr; unsigned long flags; - info = to_davinci_nand(mtd); + info = to_davinci_nand(nand_to_mtd(chip)); /* Reset ECC hardware */ - nand_davinci_readecc_1bit(mtd); + nand_davinci_readecc_1bit(nand_to_mtd(chip)); spin_lock_irqsave(&davinci_nand_lock, flags); @@ -170,10 +169,10 @@ static void nand_davinci_hwctl_1bit(struct mtd_info *mtd, int mode) /* * Read hardware ECC value and pack into three bytes */ -static int nand_davinci_calculate_1bit(struct mtd_info *mtd, - const u_char *dat, u_char *ecc_code) +static int nand_davinci_calculate_1bit(struct nand_chip *chip, + const u_char *dat, u_char *ecc_code) { - unsigned int ecc_val = nand_davinci_readecc_1bit(mtd); + unsigned int ecc_val = nand_davinci_readecc_1bit(nand_to_mtd(chip)); unsigned int ecc24 = (ecc_val & 0x0fff) | ((ecc_val & 0x0fff0000) >> 4); /* invert so that erased block ecc is correct */ @@ -185,10 +184,9 @@ static int nand_davinci_calculate_1bit(struct mtd_info *mtd, return 0; } -static int nand_davinci_correct_1bit(struct mtd_info *mtd, u_char *dat, +static int nand_davinci_correct_1bit(struct nand_chip *chip, u_char *dat, u_char *read_ecc, u_char *calc_ecc) { - struct nand_chip *chip = mtd_to_nand(mtd); uint32_t eccNand = read_ecc[0] | (read_ecc[1] << 8) | (read_ecc[2] << 16); uint32_t eccCalc = calc_ecc[0] | (calc_ecc[1] << 8) | @@ -231,9 +229,9 @@ static int nand_davinci_correct_1bit(struct mtd_info *mtd, u_char *dat, * OOB without recomputing ECC. */ -static void nand_davinci_hwctl_4bit(struct mtd_info *mtd, int mode) +static void nand_davinci_hwctl_4bit(struct nand_chip *chip, int mode) { - struct davinci_nand_info *info = to_davinci_nand(mtd); + struct davinci_nand_info *info = to_davinci_nand(nand_to_mtd(chip)); unsigned long flags; u32 val; @@ -266,10 +264,10 @@ nand_davinci_readecc_4bit(struct davinci_nand_info *info, u32 code[4]) } /* Terminate read ECC; or return ECC (as bytes) of data written to NAND. */ -static int nand_davinci_calculate_4bit(struct mtd_info *mtd, - const u_char *dat, u_char *ecc_code) +static int nand_davinci_calculate_4bit(struct nand_chip *chip, + const u_char *dat, u_char *ecc_code) { - struct davinci_nand_info *info = to_davinci_nand(mtd); + struct davinci_nand_info *info = to_davinci_nand(nand_to_mtd(chip)); u32 raw_ecc[4], *p; unsigned i; @@ -303,11 +301,11 @@ static int nand_davinci_calculate_4bit(struct mtd_info *mtd, /* Correct up to 4 bits in data we just read, using state left in the * hardware plus the ecc_code computed when it was first written. */ -static int nand_davinci_correct_4bit(struct mtd_info *mtd, - u_char *data, u_char *ecc_code, u_char *null) +static int nand_davinci_correct_4bit(struct nand_chip *chip, u_char *data, + u_char *ecc_code, u_char *null) { int i; - struct davinci_nand_info *info = to_davinci_nand(mtd); + struct davinci_nand_info *info = to_davinci_nand(nand_to_mtd(chip)); unsigned short ecc10[8]; unsigned short *ecc16; u32 syndrome[4]; @@ -436,38 +434,35 @@ correct: * the two LSBs for NAND access ... so we can issue 32-bit reads/writes * and have that transparently morphed into multiple NAND operations. */ -static void nand_davinci_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) +static void nand_davinci_read_buf(struct nand_chip *chip, uint8_t *buf, + int len) { - struct nand_chip *chip = mtd_to_nand(mtd); - if ((0x03 & ((uintptr_t)buf)) == 0 && (0x03 & len) == 0) - ioread32_rep(chip->IO_ADDR_R, buf, len >> 2); + ioread32_rep(chip->legacy.IO_ADDR_R, buf, len >> 2); else if ((0x01 & ((uintptr_t)buf)) == 0 && (0x01 & len) == 0) - ioread16_rep(chip->IO_ADDR_R, buf, len >> 1); + ioread16_rep(chip->legacy.IO_ADDR_R, buf, len >> 1); else - ioread8_rep(chip->IO_ADDR_R, buf, len); + ioread8_rep(chip->legacy.IO_ADDR_R, buf, len); } -static void nand_davinci_write_buf(struct mtd_info *mtd, - const uint8_t *buf, int len) +static void nand_davinci_write_buf(struct nand_chip *chip, const uint8_t *buf, + int len) { - struct nand_chip *chip = mtd_to_nand(mtd); - if ((0x03 & ((uintptr_t)buf)) == 0 && (0x03 & len) == 0) - iowrite32_rep(chip->IO_ADDR_R, buf, len >> 2); + iowrite32_rep(chip->legacy.IO_ADDR_R, buf, len >> 2); else if ((0x01 & ((uintptr_t)buf)) == 0 && (0x01 & len) == 0) - iowrite16_rep(chip->IO_ADDR_R, buf, len >> 1); + iowrite16_rep(chip->legacy.IO_ADDR_R, buf, len >> 1); else - iowrite8_rep(chip->IO_ADDR_R, buf, len); + iowrite8_rep(chip->legacy.IO_ADDR_R, buf, len); } /* * Check hardware register for wait status. Returns 1 if device is ready, * 0 if it is still busy. */ -static int nand_davinci_dev_ready(struct mtd_info *mtd) +static int nand_davinci_dev_ready(struct nand_chip *chip) { - struct davinci_nand_info *info = to_davinci_nand(mtd); + struct davinci_nand_info *info = to_davinci_nand(nand_to_mtd(chip)); return davinci_nand_readl(info, NANDFSR_OFFSET) & BIT(0); } @@ -764,9 +759,9 @@ static int nand_davinci_probe(struct platform_device *pdev) mtd->dev.parent = &pdev->dev; nand_set_flash_node(&info->chip, pdev->dev.of_node); - info->chip.IO_ADDR_R = vaddr; - info->chip.IO_ADDR_W = vaddr; - info->chip.chip_delay = 0; + info->chip.legacy.IO_ADDR_R = vaddr; + info->chip.legacy.IO_ADDR_W = vaddr; + info->chip.legacy.chip_delay = 0; info->chip.select_chip = nand_davinci_select_chip; /* options such as NAND_BBT_USE_FLASH */ @@ -786,12 +781,12 @@ static int nand_davinci_probe(struct platform_device *pdev) info->mask_cle = pdata->mask_cle ? : MASK_CLE; /* Set address of hardware control function */ - info->chip.cmd_ctrl = nand_davinci_hwcontrol; - info->chip.dev_ready = nand_davinci_dev_ready; + info->chip.legacy.cmd_ctrl = nand_davinci_hwcontrol; + info->chip.legacy.dev_ready = nand_davinci_dev_ready; /* Speed up buffer I/O */ - info->chip.read_buf = nand_davinci_read_buf; - info->chip.write_buf = nand_davinci_write_buf; + info->chip.legacy.read_buf = nand_davinci_read_buf; + info->chip.legacy.write_buf = nand_davinci_write_buf; /* Use board-specific ECC config */ info->chip.ecc.mode = pdata->ecc_mode; @@ -807,7 +802,7 @@ static int nand_davinci_probe(struct platform_device *pdev) /* Scan to find existence of the device(s) */ info->chip.dummy_controller.ops = &davinci_nand_controller_ops; - ret = nand_scan(mtd, pdata->mask_chipsel ? 2 : 1); + ret = nand_scan(&info->chip, pdata->mask_chipsel ? 2 : 1); if (ret < 0) { dev_dbg(&pdev->dev, "no NAND chip(s) found\n"); return ret; @@ -841,7 +836,7 @@ static int nand_davinci_remove(struct platform_device *pdev) ecc4_busy = false; spin_unlock_irq(&davinci_nand_lock); - nand_release(nand_to_mtd(&info->chip)); + nand_release(&info->chip); return 0; } diff --git a/drivers/mtd/nand/raw/denali.c b/drivers/mtd/nand/raw/denali.c index b864b93dd289..830ea247277b 100644 --- a/drivers/mtd/nand/raw/denali.c +++ b/drivers/mtd/nand/raw/denali.c @@ -1,15 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * NAND Flash Controller Device Driver * Copyright © 2009-2010, Intel Corporation and its suppliers. * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. + * Copyright (c) 2017 Socionext Inc. + * Reworked by Masahiro Yamada <yamada.masahiro@socionext.com> */ #include <linux/bitfield.h> @@ -25,9 +20,8 @@ #include "denali.h" -MODULE_LICENSE("GPL"); - #define DENALI_NAND_NAME "denali-nand" +#define DENALI_DEFAULT_OOB_SKIP_BYTES 8 /* for Indexed Addressing */ #define DENALI_INDEXED_CTRL 0x00 @@ -222,8 +216,9 @@ static uint32_t denali_check_irq(struct denali_nand_info *denali) return irq_status; } -static void denali_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) +static void denali_read_buf(struct nand_chip *chip, uint8_t *buf, int len) { + struct mtd_info *mtd = nand_to_mtd(chip); struct denali_nand_info *denali = mtd_to_denali(mtd); u32 addr = DENALI_MAP11_DATA | DENALI_BANK(denali); int i; @@ -232,9 +227,10 @@ static void denali_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) buf[i] = denali->host_read(denali, addr); } -static void denali_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len) +static void denali_write_buf(struct nand_chip *chip, const uint8_t *buf, + int len) { - struct denali_nand_info *denali = mtd_to_denali(mtd); + struct denali_nand_info *denali = mtd_to_denali(nand_to_mtd(chip)); u32 addr = DENALI_MAP11_DATA | DENALI_BANK(denali); int i; @@ -242,9 +238,9 @@ static void denali_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len) denali->host_write(denali, addr, buf[i]); } -static void denali_read_buf16(struct mtd_info *mtd, uint8_t *buf, int len) +static void denali_read_buf16(struct nand_chip *chip, uint8_t *buf, int len) { - struct denali_nand_info *denali = mtd_to_denali(mtd); + struct denali_nand_info *denali = mtd_to_denali(nand_to_mtd(chip)); u32 addr = DENALI_MAP11_DATA | DENALI_BANK(denali); uint16_t *buf16 = (uint16_t *)buf; int i; @@ -253,10 +249,10 @@ static void denali_read_buf16(struct mtd_info *mtd, uint8_t *buf, int len) buf16[i] = denali->host_read(denali, addr); } -static void denali_write_buf16(struct mtd_info *mtd, const uint8_t *buf, +static void denali_write_buf16(struct nand_chip *chip, const uint8_t *buf, int len) { - struct denali_nand_info *denali = mtd_to_denali(mtd); + struct denali_nand_info *denali = mtd_to_denali(nand_to_mtd(chip)); u32 addr = DENALI_MAP11_DATA | DENALI_BANK(denali); const uint16_t *buf16 = (const uint16_t *)buf; int i; @@ -265,32 +261,23 @@ static void denali_write_buf16(struct mtd_info *mtd, const uint8_t *buf, denali->host_write(denali, addr, buf16[i]); } -static uint8_t denali_read_byte(struct mtd_info *mtd) +static uint8_t denali_read_byte(struct nand_chip *chip) { uint8_t byte; - denali_read_buf(mtd, &byte, 1); + denali_read_buf(chip, &byte, 1); return byte; } -static void denali_write_byte(struct mtd_info *mtd, uint8_t byte) -{ - denali_write_buf(mtd, &byte, 1); -} - -static uint16_t denali_read_word(struct mtd_info *mtd) +static void denali_write_byte(struct nand_chip *chip, uint8_t byte) { - uint16_t word; - - denali_read_buf16(mtd, (uint8_t *)&word, 2); - - return word; + denali_write_buf(chip, &byte, 1); } -static void denali_cmd_ctrl(struct mtd_info *mtd, int dat, unsigned int ctrl) +static void denali_cmd_ctrl(struct nand_chip *chip, int dat, unsigned int ctrl) { - struct denali_nand_info *denali = mtd_to_denali(mtd); + struct denali_nand_info *denali = mtd_to_denali(nand_to_mtd(chip)); uint32_t type; if (ctrl & NAND_CLE) @@ -301,7 +288,8 @@ static void denali_cmd_ctrl(struct mtd_info *mtd, int dat, unsigned int ctrl) return; /* - * Some commands are followed by chip->dev_ready or chip->waitfunc. + * Some commands are followed by chip->legacy.dev_ready or + * chip->legacy.waitfunc. * irq_status must be cleared here to catch the R/B# interrupt later. */ if (ctrl & NAND_CTRL_CHANGE) @@ -310,9 +298,9 @@ static void denali_cmd_ctrl(struct mtd_info *mtd, int dat, unsigned int ctrl) denali->host_write(denali, DENALI_BANK(denali) | type, dat); } -static int denali_dev_ready(struct mtd_info *mtd) +static int denali_dev_ready(struct nand_chip *chip) { - struct denali_nand_info *denali = mtd_to_denali(mtd); + struct denali_nand_info *denali = mtd_to_denali(nand_to_mtd(chip)); return !!(denali_check_irq(denali) & INTR__INT_ACT); } @@ -698,9 +686,10 @@ static void denali_oob_xfer(struct mtd_info *mtd, struct nand_chip *chip, false); } -static int denali_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int denali_read_page_raw(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct denali_nand_info *denali = mtd_to_denali(mtd); int writesize = mtd->writesize; int oobsize = mtd->oobsize; @@ -773,17 +762,18 @@ static int denali_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip, return 0; } -static int denali_read_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int denali_read_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + denali_oob_xfer(mtd, chip, page, 0); return 0; } -static int denali_write_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int denali_write_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct denali_nand_info *denali = mtd_to_denali(mtd); denali_reset_irq(denali); @@ -793,9 +783,10 @@ static int denali_write_oob(struct mtd_info *mtd, struct nand_chip *chip, return nand_prog_page_end_op(chip); } -static int denali_read_page(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int denali_read_page(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct denali_nand_info *denali = mtd_to_denali(mtd); unsigned long uncor_ecc_flags = 0; int stat = 0; @@ -814,7 +805,7 @@ static int denali_read_page(struct mtd_info *mtd, struct nand_chip *chip, return stat; if (uncor_ecc_flags) { - ret = denali_read_oob(mtd, chip, page); + ret = denali_read_oob(chip, page); if (ret) return ret; @@ -825,9 +816,10 @@ static int denali_read_page(struct mtd_info *mtd, struct nand_chip *chip, return stat; } -static int denali_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, int page) +static int denali_write_page_raw(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct denali_nand_info *denali = mtd_to_denali(mtd); int writesize = mtd->writesize; int oobsize = mtd->oobsize; @@ -903,25 +895,26 @@ static int denali_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip, return denali_data_xfer(denali, tmp_buf, size, page, 1, 1); } -static int denali_write_page(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, int page) +static int denali_write_page(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct denali_nand_info *denali = mtd_to_denali(mtd); return denali_data_xfer(denali, (void *)buf, mtd->writesize, page, 0, 1); } -static void denali_select_chip(struct mtd_info *mtd, int chip) +static void denali_select_chip(struct nand_chip *chip, int cs) { - struct denali_nand_info *denali = mtd_to_denali(mtd); + struct denali_nand_info *denali = mtd_to_denali(nand_to_mtd(chip)); - denali->active_bank = chip; + denali->active_bank = cs; } -static int denali_waitfunc(struct mtd_info *mtd, struct nand_chip *chip) +static int denali_waitfunc(struct nand_chip *chip) { - struct denali_nand_info *denali = mtd_to_denali(mtd); + struct denali_nand_info *denali = mtd_to_denali(nand_to_mtd(chip)); uint32_t irq_status; /* R/B# pin transitioned from low to high? */ @@ -930,9 +923,9 @@ static int denali_waitfunc(struct mtd_info *mtd, struct nand_chip *chip) return irq_status & INTR__INT_ACT ? 0 : NAND_STATUS_FAIL; } -static int denali_erase(struct mtd_info *mtd, int page) +static int denali_erase(struct nand_chip *chip, int page) { - struct denali_nand_info *denali = mtd_to_denali(mtd); + struct denali_nand_info *denali = mtd_to_denali(nand_to_mtd(chip)); uint32_t irq_status; denali_reset_irq(denali); @@ -947,10 +940,10 @@ static int denali_erase(struct mtd_info *mtd, int page) return irq_status & INTR__ERASE_COMP ? 0 : -EIO; } -static int denali_setup_data_interface(struct mtd_info *mtd, int chipnr, +static int denali_setup_data_interface(struct nand_chip *chip, int chipnr, const struct nand_data_interface *conf) { - struct denali_nand_info *denali = mtd_to_denali(mtd); + struct denali_nand_info *denali = mtd_to_denali(nand_to_mtd(chip)); const struct nand_sdr_timings *timings; unsigned long t_x, mult_x; int acc_clks, re_2_we, re_2_re, we_2_re, addr_2_data; @@ -1105,12 +1098,17 @@ static void denali_hw_init(struct denali_nand_info *denali) denali->revision = swab16(ioread32(denali->reg + REVISION)); /* - * tell driver how many bit controller will skip before - * writing ECC code in OOB, this register may be already - * set by firmware. So we read this value out. - * if this value is 0, just let it be. + * Set how many bytes should be skipped before writing data in OOB. + * If a non-zero value has already been set (by firmware or something), + * just use it. Otherwise, set the driver default. */ denali->oob_skip_bytes = ioread32(denali->reg + SPARE_AREA_SKIP_BYTES); + if (!denali->oob_skip_bytes) { + denali->oob_skip_bytes = DENALI_DEFAULT_OOB_SKIP_BYTES; + iowrite32(denali->oob_skip_bytes, + denali->reg + SPARE_AREA_SKIP_BYTES); + } + denali_detect_max_banks(denali); iowrite32(0x0F, denali->reg + RB_PIN_ENABLED); iowrite32(CHIP_EN_DONT_CARE__FLAG, denali->reg + CHIP_ENABLE_DONT_CARE); @@ -1277,11 +1275,11 @@ static int denali_attach_chip(struct nand_chip *chip) mtd_set_ooblayout(mtd, &denali_ooblayout_ops); if (chip->options & NAND_BUSWIDTH_16) { - chip->read_buf = denali_read_buf16; - chip->write_buf = denali_write_buf16; + chip->legacy.read_buf = denali_read_buf16; + chip->legacy.write_buf = denali_write_buf16; } else { - chip->read_buf = denali_read_buf; - chip->write_buf = denali_write_buf; + chip->legacy.read_buf = denali_read_buf; + chip->legacy.write_buf = denali_write_buf; } chip->ecc.read_page = denali_read_page; chip->ecc.read_page_raw = denali_read_page_raw; @@ -1289,7 +1287,7 @@ static int denali_attach_chip(struct nand_chip *chip) chip->ecc.write_page_raw = denali_write_page_raw; chip->ecc.read_oob = denali_read_oob; chip->ecc.write_oob = denali_write_oob; - chip->erase = denali_erase; + chip->legacy.erase = denali_erase; ret = denali_multidev_fixup(denali); if (ret) @@ -1358,12 +1356,11 @@ int denali_init(struct denali_nand_info *denali) mtd->name = "denali-nand"; chip->select_chip = denali_select_chip; - chip->read_byte = denali_read_byte; - chip->write_byte = denali_write_byte; - chip->read_word = denali_read_word; - chip->cmd_ctrl = denali_cmd_ctrl; - chip->dev_ready = denali_dev_ready; - chip->waitfunc = denali_waitfunc; + chip->legacy.read_byte = denali_read_byte; + chip->legacy.write_byte = denali_write_byte; + chip->legacy.cmd_ctrl = denali_cmd_ctrl; + chip->legacy.dev_ready = denali_dev_ready; + chip->legacy.waitfunc = denali_waitfunc; if (features & FEATURES__INDEX_ADDR) { denali->host_read = denali_indexed_read; @@ -1378,7 +1375,7 @@ int denali_init(struct denali_nand_info *denali) chip->setup_data_interface = denali_setup_data_interface; chip->dummy_controller.ops = &denali_controller_ops; - ret = nand_scan(mtd, denali->max_banks); + ret = nand_scan(chip, denali->max_banks); if (ret) goto disable_irq; @@ -1401,9 +1398,11 @@ EXPORT_SYMBOL(denali_init); void denali_remove(struct denali_nand_info *denali) { - struct mtd_info *mtd = nand_to_mtd(&denali->nand); - - nand_release(mtd); + nand_release(&denali->nand); denali_disable_irq(denali); } EXPORT_SYMBOL(denali_remove); + +MODULE_DESCRIPTION("Driver core for Denali NAND controller"); +MODULE_AUTHOR("Intel Corporation and its suppliers"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/mtd/nand/raw/denali.h b/drivers/mtd/nand/raw/denali.h index 1f8feaf924eb..57a5498f58bb 100644 --- a/drivers/mtd/nand/raw/denali.h +++ b/drivers/mtd/nand/raw/denali.h @@ -1,15 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * NAND Flash Controller Device Driver * Copyright (c) 2009 - 2010, Intel Corporation and its suppliers. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. */ #ifndef __DENALI_H__ diff --git a/drivers/mtd/nand/raw/denali_dt.c b/drivers/mtd/nand/raw/denali_dt.c index 0faaad032e5f..7c6a8a426606 100644 --- a/drivers/mtd/nand/raw/denali_dt.c +++ b/drivers/mtd/nand/raw/denali_dt.c @@ -1,16 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * NAND Flash Controller Device Driver for DT * * Copyright © 2011, Picochip. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. */ #include <linux/clk.h> @@ -202,6 +194,6 @@ static struct platform_driver denali_dt_driver = { }; module_platform_driver(denali_dt_driver); -MODULE_LICENSE("GPL"); +MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Jamie Iles"); MODULE_DESCRIPTION("DT driver for Denali NAND controller"); diff --git a/drivers/mtd/nand/raw/denali_pci.c b/drivers/mtd/nand/raw/denali_pci.c index 7c8efc4c7bdf..48e9ac54ad53 100644 --- a/drivers/mtd/nand/raw/denali_pci.c +++ b/drivers/mtd/nand/raw/denali_pci.c @@ -1,15 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * NAND Flash Controller Device Driver * Copyright © 2009-2010, Intel Corporation and its suppliers. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. */ #include <linux/errno.h> diff --git a/drivers/mtd/nand/raw/diskonchip.c b/drivers/mtd/nand/raw/diskonchip.c index 3c46188dd6d2..3a4c373affab 100644 --- a/drivers/mtd/nand/raw/diskonchip.c +++ b/drivers/mtd/nand/raw/diskonchip.c @@ -83,9 +83,9 @@ static u_char empty_write_ecc[6] = { 0x4b, 0x00, 0xe2, 0x0e, 0x93, 0xf7 }; #define DoC_is_Millennium(doc) ((doc)->ChipID == DOC_ChipID_DocMil) #define DoC_is_2000(doc) ((doc)->ChipID == DOC_ChipID_Doc2k) -static void doc200x_hwcontrol(struct mtd_info *mtd, int cmd, +static void doc200x_hwcontrol(struct nand_chip *this, int cmd, unsigned int bitmask); -static void doc200x_select_chip(struct mtd_info *mtd, int chip); +static void doc200x_select_chip(struct nand_chip *this, int chip); static int debug = 0; module_param(debug, int, 0); @@ -290,9 +290,8 @@ static inline int DoC_WaitReady(struct doc_priv *doc) return ret; } -static void doc2000_write_byte(struct mtd_info *mtd, u_char datum) +static void doc2000_write_byte(struct nand_chip *this, u_char datum) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; @@ -302,9 +301,8 @@ static void doc2000_write_byte(struct mtd_info *mtd, u_char datum) WriteDOC(datum, docptr, 2k_CDSN_IO); } -static u_char doc2000_read_byte(struct mtd_info *mtd) +static u_char doc2000_read_byte(struct nand_chip *this) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; u_char ret; @@ -317,9 +315,9 @@ static u_char doc2000_read_byte(struct mtd_info *mtd) return ret; } -static void doc2000_writebuf(struct mtd_info *mtd, const u_char *buf, int len) +static void doc2000_writebuf(struct nand_chip *this, const u_char *buf, + int len) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; int i; @@ -334,9 +332,8 @@ static void doc2000_writebuf(struct mtd_info *mtd, const u_char *buf, int len) printk("\n"); } -static void doc2000_readbuf(struct mtd_info *mtd, u_char *buf, int len) +static void doc2000_readbuf(struct nand_chip *this, u_char *buf, int len) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; int i; @@ -344,14 +341,12 @@ static void doc2000_readbuf(struct mtd_info *mtd, u_char *buf, int len) if (debug) printk("readbuf of %d bytes: ", len); - for (i = 0; i < len; i++) { + for (i = 0; i < len; i++) buf[i] = ReadDOC(docptr, 2k_CDSN_IO + i); - } } -static void doc2000_readbuf_dword(struct mtd_info *mtd, u_char *buf, int len) +static void doc2000_readbuf_dword(struct nand_chip *this, u_char *buf, int len) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; int i; @@ -376,19 +371,19 @@ static uint16_t __init doc200x_ident_chip(struct mtd_info *mtd, int nr) struct doc_priv *doc = nand_get_controller_data(this); uint16_t ret; - doc200x_select_chip(mtd, nr); - doc200x_hwcontrol(mtd, NAND_CMD_READID, + doc200x_select_chip(this, nr); + doc200x_hwcontrol(this, NAND_CMD_READID, NAND_CTRL_CLE | NAND_CTRL_CHANGE); - doc200x_hwcontrol(mtd, 0, NAND_CTRL_ALE | NAND_CTRL_CHANGE); - doc200x_hwcontrol(mtd, NAND_CMD_NONE, NAND_NCE | NAND_CTRL_CHANGE); + doc200x_hwcontrol(this, 0, NAND_CTRL_ALE | NAND_CTRL_CHANGE); + doc200x_hwcontrol(this, NAND_CMD_NONE, NAND_NCE | NAND_CTRL_CHANGE); /* We can't use dev_ready here, but at least we wait for the * command to complete */ udelay(50); - ret = this->read_byte(mtd) << 8; - ret |= this->read_byte(mtd); + ret = this->legacy.read_byte(this) << 8; + ret |= this->legacy.read_byte(this); if (doc->ChipID == DOC_ChipID_Doc2k && try_dword && !nr) { /* First chip probe. See if we get same results by 32-bit access */ @@ -398,10 +393,10 @@ static uint16_t __init doc200x_ident_chip(struct mtd_info *mtd, int nr) } ident; void __iomem *docptr = doc->virtadr; - doc200x_hwcontrol(mtd, NAND_CMD_READID, + doc200x_hwcontrol(this, NAND_CMD_READID, NAND_CTRL_CLE | NAND_CTRL_CHANGE); - doc200x_hwcontrol(mtd, 0, NAND_CTRL_ALE | NAND_CTRL_CHANGE); - doc200x_hwcontrol(mtd, NAND_CMD_NONE, + doc200x_hwcontrol(this, 0, NAND_CTRL_ALE | NAND_CTRL_CHANGE); + doc200x_hwcontrol(this, NAND_CMD_NONE, NAND_NCE | NAND_CTRL_CHANGE); udelay(50); @@ -409,7 +404,7 @@ static uint16_t __init doc200x_ident_chip(struct mtd_info *mtd, int nr) ident.dword = readl(docptr + DoC_2k_CDSN_IO); if (((ident.byte[0] << 8) | ident.byte[1]) == ret) { pr_info("DiskOnChip 2000 responds to DWORD access\n"); - this->read_buf = &doc2000_readbuf_dword; + this->legacy.read_buf = &doc2000_readbuf_dword; } } @@ -438,7 +433,7 @@ static void __init doc2000_count_chips(struct mtd_info *mtd) pr_debug("Detected %d chips per floor.\n", i); } -static int doc200x_wait(struct mtd_info *mtd, struct nand_chip *this) +static int doc200x_wait(struct nand_chip *this) { struct doc_priv *doc = nand_get_controller_data(this); @@ -447,14 +442,13 @@ static int doc200x_wait(struct mtd_info *mtd, struct nand_chip *this) DoC_WaitReady(doc); nand_status_op(this, NULL); DoC_WaitReady(doc); - status = (int)this->read_byte(mtd); + status = (int)this->legacy.read_byte(this); return status; } -static void doc2001_write_byte(struct mtd_info *mtd, u_char datum) +static void doc2001_write_byte(struct nand_chip *this, u_char datum) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; @@ -463,9 +457,8 @@ static void doc2001_write_byte(struct mtd_info *mtd, u_char datum) WriteDOC(datum, docptr, WritePipeTerm); } -static u_char doc2001_read_byte(struct mtd_info *mtd) +static u_char doc2001_read_byte(struct nand_chip *this) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; @@ -477,9 +470,8 @@ static u_char doc2001_read_byte(struct mtd_info *mtd) return ReadDOC(docptr, LastDataRead); } -static void doc2001_writebuf(struct mtd_info *mtd, const u_char *buf, int len) +static void doc2001_writebuf(struct nand_chip *this, const u_char *buf, int len) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; int i; @@ -490,9 +482,8 @@ static void doc2001_writebuf(struct mtd_info *mtd, const u_char *buf, int len) WriteDOC(0x00, docptr, WritePipeTerm); } -static void doc2001_readbuf(struct mtd_info *mtd, u_char *buf, int len) +static void doc2001_readbuf(struct nand_chip *this, u_char *buf, int len) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; int i; @@ -507,9 +498,8 @@ static void doc2001_readbuf(struct mtd_info *mtd, u_char *buf, int len) buf[i] = ReadDOC(docptr, LastDataRead); } -static u_char doc2001plus_read_byte(struct mtd_info *mtd) +static u_char doc2001plus_read_byte(struct nand_chip *this) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; u_char ret; @@ -522,9 +512,8 @@ static u_char doc2001plus_read_byte(struct mtd_info *mtd) return ret; } -static void doc2001plus_writebuf(struct mtd_info *mtd, const u_char *buf, int len) +static void doc2001plus_writebuf(struct nand_chip *this, const u_char *buf, int len) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; int i; @@ -540,9 +529,8 @@ static void doc2001plus_writebuf(struct mtd_info *mtd, const u_char *buf, int le printk("\n"); } -static void doc2001plus_readbuf(struct mtd_info *mtd, u_char *buf, int len) +static void doc2001plus_readbuf(struct nand_chip *this, u_char *buf, int len) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; int i; @@ -571,9 +559,8 @@ static void doc2001plus_readbuf(struct mtd_info *mtd, u_char *buf, int len) printk("\n"); } -static void doc2001plus_select_chip(struct mtd_info *mtd, int chip) +static void doc2001plus_select_chip(struct nand_chip *this, int chip) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; int floor = 0; @@ -598,9 +585,8 @@ static void doc2001plus_select_chip(struct mtd_info *mtd, int chip) doc->curfloor = floor; } -static void doc200x_select_chip(struct mtd_info *mtd, int chip) +static void doc200x_select_chip(struct nand_chip *this, int chip) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; int floor = 0; @@ -615,12 +601,12 @@ static void doc200x_select_chip(struct mtd_info *mtd, int chip) chip -= (floor * doc->chips_per_floor); /* 11.4.4 -- deassert CE before changing chip */ - doc200x_hwcontrol(mtd, NAND_CMD_NONE, 0 | NAND_CTRL_CHANGE); + doc200x_hwcontrol(this, NAND_CMD_NONE, 0 | NAND_CTRL_CHANGE); WriteDOC(floor, docptr, FloorSelect); WriteDOC(chip, docptr, CDSNDeviceSelect); - doc200x_hwcontrol(mtd, NAND_CMD_NONE, NAND_NCE | NAND_CTRL_CHANGE); + doc200x_hwcontrol(this, NAND_CMD_NONE, NAND_NCE | NAND_CTRL_CHANGE); doc->curchip = chip; doc->curfloor = floor; @@ -628,10 +614,9 @@ static void doc200x_select_chip(struct mtd_info *mtd, int chip) #define CDSN_CTRL_MSK (CDSN_CTRL_CE | CDSN_CTRL_CLE | CDSN_CTRL_ALE) -static void doc200x_hwcontrol(struct mtd_info *mtd, int cmd, +static void doc200x_hwcontrol(struct nand_chip *this, int cmd, unsigned int ctrl) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; @@ -646,15 +631,16 @@ static void doc200x_hwcontrol(struct mtd_info *mtd, int cmd, } if (cmd != NAND_CMD_NONE) { if (DoC_is_2000(doc)) - doc2000_write_byte(mtd, cmd); + doc2000_write_byte(this, cmd); else - doc2001_write_byte(mtd, cmd); + doc2001_write_byte(this, cmd); } } -static void doc2001plus_command(struct mtd_info *mtd, unsigned command, int column, int page_addr) +static void doc2001plus_command(struct nand_chip *this, unsigned command, + int column, int page_addr) { - struct nand_chip *this = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(this); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; @@ -729,13 +715,13 @@ static void doc2001plus_command(struct mtd_info *mtd, unsigned command, int colu return; case NAND_CMD_RESET: - if (this->dev_ready) + if (this->legacy.dev_ready) break; - udelay(this->chip_delay); + udelay(this->legacy.chip_delay); WriteDOC(NAND_CMD_STATUS, docptr, Mplus_FlashCmd); WriteDOC(0, docptr, Mplus_WritePipeTerm); WriteDOC(0, docptr, Mplus_WritePipeTerm); - while (!(this->read_byte(mtd) & 0x40)) ; + while (!(this->legacy.read_byte(this) & 0x40)) ; return; /* This applies to read commands */ @@ -744,8 +730,8 @@ static void doc2001plus_command(struct mtd_info *mtd, unsigned command, int colu * If we don't have access to the busy pin, we apply the given * command delay */ - if (!this->dev_ready) { - udelay(this->chip_delay); + if (!this->legacy.dev_ready) { + udelay(this->legacy.chip_delay); return; } } @@ -754,12 +740,11 @@ static void doc2001plus_command(struct mtd_info *mtd, unsigned command, int colu * any case on any machine. */ ndelay(100); /* wait until command is processed */ - while (!this->dev_ready(mtd)) ; + while (!this->legacy.dev_ready(this)) ; } -static int doc200x_dev_ready(struct mtd_info *mtd) +static int doc200x_dev_ready(struct nand_chip *this) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; @@ -790,16 +775,15 @@ static int doc200x_dev_ready(struct mtd_info *mtd) } } -static int doc200x_block_bad(struct mtd_info *mtd, loff_t ofs) +static int doc200x_block_bad(struct nand_chip *this, loff_t ofs) { /* This is our last resort if we couldn't find or create a BBT. Just pretend all blocks are good. */ return 0; } -static void doc200x_enable_hwecc(struct mtd_info *mtd, int mode) +static void doc200x_enable_hwecc(struct nand_chip *this, int mode) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; @@ -816,9 +800,8 @@ static void doc200x_enable_hwecc(struct mtd_info *mtd, int mode) } } -static void doc2001plus_enable_hwecc(struct mtd_info *mtd, int mode) +static void doc2001plus_enable_hwecc(struct nand_chip *this, int mode) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; @@ -836,9 +819,9 @@ static void doc2001plus_enable_hwecc(struct mtd_info *mtd, int mode) } /* This code is only called on write */ -static int doc200x_calculate_ecc(struct mtd_info *mtd, const u_char *dat, unsigned char *ecc_code) +static int doc200x_calculate_ecc(struct nand_chip *this, const u_char *dat, + unsigned char *ecc_code) { - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; int i; @@ -895,11 +878,10 @@ static int doc200x_calculate_ecc(struct mtd_info *mtd, const u_char *dat, unsign return 0; } -static int doc200x_correct_data(struct mtd_info *mtd, u_char *dat, +static int doc200x_correct_data(struct nand_chip *this, u_char *dat, u_char *read_ecc, u_char *isnull) { int i, ret = 0; - struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); void __iomem *docptr = doc->virtadr; uint8_t calc_ecc[6]; @@ -1357,9 +1339,9 @@ static inline int __init doc2000_init(struct mtd_info *mtd) struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); - this->read_byte = doc2000_read_byte; - this->write_buf = doc2000_writebuf; - this->read_buf = doc2000_readbuf; + this->legacy.read_byte = doc2000_read_byte; + this->legacy.write_buf = doc2000_writebuf; + this->legacy.read_buf = doc2000_readbuf; doc->late_init = nftl_scan_bbt; doc->CDSNControl = CDSN_CTRL_FLASH_IO | CDSN_CTRL_ECC_IO; @@ -1373,9 +1355,9 @@ static inline int __init doc2001_init(struct mtd_info *mtd) struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); - this->read_byte = doc2001_read_byte; - this->write_buf = doc2001_writebuf; - this->read_buf = doc2001_readbuf; + this->legacy.read_byte = doc2001_read_byte; + this->legacy.write_buf = doc2001_writebuf; + this->legacy.read_buf = doc2001_readbuf; ReadDOC(doc->virtadr, ChipID); ReadDOC(doc->virtadr, ChipID); @@ -1403,13 +1385,13 @@ static inline int __init doc2001plus_init(struct mtd_info *mtd) struct nand_chip *this = mtd_to_nand(mtd); struct doc_priv *doc = nand_get_controller_data(this); - this->read_byte = doc2001plus_read_byte; - this->write_buf = doc2001plus_writebuf; - this->read_buf = doc2001plus_readbuf; + this->legacy.read_byte = doc2001plus_read_byte; + this->legacy.write_buf = doc2001plus_writebuf; + this->legacy.read_buf = doc2001plus_readbuf; doc->late_init = inftl_scan_bbt; - this->cmd_ctrl = NULL; + this->legacy.cmd_ctrl = NULL; this->select_chip = doc2001plus_select_chip; - this->cmdfunc = doc2001plus_command; + this->legacy.cmdfunc = doc2001plus_command; this->ecc.hwctl = doc2001plus_enable_hwecc; doc->chips_per_floor = 1; @@ -1587,10 +1569,10 @@ static int __init doc_probe(unsigned long physadr) nand_set_controller_data(nand, doc); nand->select_chip = doc200x_select_chip; - nand->cmd_ctrl = doc200x_hwcontrol; - nand->dev_ready = doc200x_dev_ready; - nand->waitfunc = doc200x_wait; - nand->block_bad = doc200x_block_bad; + nand->legacy.cmd_ctrl = doc200x_hwcontrol; + nand->legacy.dev_ready = doc200x_dev_ready; + nand->legacy.waitfunc = doc200x_wait; + nand->legacy.block_bad = doc200x_block_bad; nand->ecc.hwctl = doc200x_enable_hwecc; nand->ecc.calculate = doc200x_calculate_ecc; nand->ecc.correct = doc200x_correct_data; @@ -1620,14 +1602,14 @@ static int __init doc_probe(unsigned long physadr) else numchips = doc2001_init(mtd); - if ((ret = nand_scan(mtd, numchips)) || (ret = doc->late_init(mtd))) { + if ((ret = nand_scan(nand, numchips)) || (ret = doc->late_init(mtd))) { /* DBB note: i believe nand_release is necessary here, as buffers may have been allocated in nand_base. Check with Thomas. FIX ME! */ /* nand_release will call mtd_device_unregister, but we haven't yet added it. This is handled without incident by mtd_device_unregister, as far as I can tell. */ - nand_release(mtd); + nand_release(nand); goto fail; } @@ -1662,7 +1644,7 @@ static void release_nanddoc(void) doc = nand_get_controller_data(nand); nextmtd = doc->nextdoc; - nand_release(mtd); + nand_release(nand); iounmap(doc->virtadr); release_mem_region(doc->physadr, DOC_IOREMAP_LEN); free_rs(doc->rs_decoder); diff --git a/drivers/mtd/nand/raw/docg4.c b/drivers/mtd/nand/raw/docg4.c deleted file mode 100644 index 427fcbc1b71c..000000000000 --- a/drivers/mtd/nand/raw/docg4.c +++ /dev/null @@ -1,1442 +0,0 @@ -/* - * Copyright © 2012 Mike Dunn <mikedunn@newsguy.com> - * - * mtd nand driver for M-Systems DiskOnChip G4 - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * Tested on the Palm Treo 680. The G4 is also present on Toshiba Portege, Asus - * P526, some HTC smartphones (Wizard, Prophet, ...), O2 XDA Zinc, maybe others. - * Should work on these as well. Let me know! - * - * TODO: - * - * Mechanism for management of password-protected areas - * - * Hamming ecc when reading oob only - * - * According to the M-Sys documentation, this device is also available in a - * "dual-die" configuration having a 256MB capacity, but no mechanism for - * detecting this variant is documented. Currently this driver assumes 128MB - * capacity. - * - * Support for multiple cascaded devices ("floors"). Not sure which gadgets - * contain multiple G4s in a cascaded configuration, if any. - * - */ - -#include <linux/kernel.h> -#include <linux/slab.h> -#include <linux/init.h> -#include <linux/string.h> -#include <linux/sched.h> -#include <linux/delay.h> -#include <linux/module.h> -#include <linux/export.h> -#include <linux/platform_device.h> -#include <linux/io.h> -#include <linux/bitops.h> -#include <linux/mtd/partitions.h> -#include <linux/mtd/mtd.h> -#include <linux/mtd/rawnand.h> -#include <linux/bch.h> -#include <linux/bitrev.h> -#include <linux/jiffies.h> - -/* - * In "reliable mode" consecutive 2k pages are used in parallel (in some - * fashion) to store the same data. The data can be read back from the - * even-numbered pages in the normal manner; odd-numbered pages will appear to - * contain junk. Systems that boot from the docg4 typically write the secondary - * program loader (SPL) code in this mode. The SPL is loaded by the initial - * program loader (IPL, stored in the docg4's 2k NOR-like region that is mapped - * to the reset vector address). This module parameter enables you to use this - * driver to write the SPL. When in this mode, no more than 2k of data can be - * written at a time, because the addresses do not increment in the normal - * manner, and the starting offset must be within an even-numbered 2k region; - * i.e., invalid starting offsets are 0x800, 0xa00, 0xc00, 0xe00, 0x1800, - * 0x1a00, ... Reliable mode is a special case and should not be used unless - * you know what you're doing. - */ -static bool reliable_mode; -module_param(reliable_mode, bool, 0); -MODULE_PARM_DESC(reliable_mode, "pages are programmed in reliable mode"); - -/* - * You'll want to ignore badblocks if you're reading a partition that contains - * data written by the TrueFFS library (i.e., by PalmOS, Windows, etc), since - * it does not use mtd nand's method for marking bad blocks (using oob area). - * This will also skip the check of the "page written" flag. - */ -static bool ignore_badblocks; -module_param(ignore_badblocks, bool, 0); -MODULE_PARM_DESC(ignore_badblocks, "no badblock checking performed"); - -struct docg4_priv { - struct mtd_info *mtd; - struct device *dev; - void __iomem *virtadr; - int status; - struct { - unsigned int command; - int column; - int page; - } last_command; - uint8_t oob_buf[16]; - uint8_t ecc_buf[7]; - int oob_page; - struct bch_control *bch; -}; - -/* - * Defines prefixed with DOCG4 are unique to the diskonchip G4. All others are - * shared with other diskonchip devices (P3, G3 at least). - * - * Functions with names prefixed with docg4_ are mtd / nand interface functions - * (though they may also be called internally). All others are internal. - */ - -#define DOC_IOSPACE_DATA 0x0800 - -/* register offsets */ -#define DOC_CHIPID 0x1000 -#define DOC_DEVICESELECT 0x100a -#define DOC_ASICMODE 0x100c -#define DOC_DATAEND 0x101e -#define DOC_NOP 0x103e - -#define DOC_FLASHSEQUENCE 0x1032 -#define DOC_FLASHCOMMAND 0x1034 -#define DOC_FLASHADDRESS 0x1036 -#define DOC_FLASHCONTROL 0x1038 -#define DOC_ECCCONF0 0x1040 -#define DOC_ECCCONF1 0x1042 -#define DOC_HAMMINGPARITY 0x1046 -#define DOC_BCH_SYNDROM(idx) (0x1048 + idx) - -#define DOC_ASICMODECONFIRM 0x1072 -#define DOC_CHIPID_INV 0x1074 -#define DOC_POWERMODE 0x107c - -#define DOCG4_MYSTERY_REG 0x1050 - -/* apparently used only to write oob bytes 6 and 7 */ -#define DOCG4_OOB_6_7 0x1052 - -/* DOC_FLASHSEQUENCE register commands */ -#define DOC_SEQ_RESET 0x00 -#define DOCG4_SEQ_PAGE_READ 0x03 -#define DOCG4_SEQ_FLUSH 0x29 -#define DOCG4_SEQ_PAGEWRITE 0x16 -#define DOCG4_SEQ_PAGEPROG 0x1e -#define DOCG4_SEQ_BLOCKERASE 0x24 -#define DOCG4_SEQ_SETMODE 0x45 - -/* DOC_FLASHCOMMAND register commands */ -#define DOCG4_CMD_PAGE_READ 0x00 -#define DOC_CMD_ERASECYCLE2 0xd0 -#define DOCG4_CMD_FLUSH 0x70 -#define DOCG4_CMD_READ2 0x30 -#define DOC_CMD_PROG_BLOCK_ADDR 0x60 -#define DOCG4_CMD_PAGEWRITE 0x80 -#define DOC_CMD_PROG_CYCLE2 0x10 -#define DOCG4_CMD_FAST_MODE 0xa3 /* functionality guessed */ -#define DOC_CMD_RELIABLE_MODE 0x22 -#define DOC_CMD_RESET 0xff - -/* DOC_POWERMODE register bits */ -#define DOC_POWERDOWN_READY 0x80 - -/* DOC_FLASHCONTROL register bits */ -#define DOC_CTRL_CE 0x10 -#define DOC_CTRL_UNKNOWN 0x40 -#define DOC_CTRL_FLASHREADY 0x01 - -/* DOC_ECCCONF0 register bits */ -#define DOC_ECCCONF0_READ_MODE 0x8000 -#define DOC_ECCCONF0_UNKNOWN 0x2000 -#define DOC_ECCCONF0_ECC_ENABLE 0x1000 -#define DOC_ECCCONF0_DATA_BYTES_MASK 0x07ff - -/* DOC_ECCCONF1 register bits */ -#define DOC_ECCCONF1_BCH_SYNDROM_ERR 0x80 -#define DOC_ECCCONF1_ECC_ENABLE 0x07 -#define DOC_ECCCONF1_PAGE_IS_WRITTEN 0x20 - -/* DOC_ASICMODE register bits */ -#define DOC_ASICMODE_RESET 0x00 -#define DOC_ASICMODE_NORMAL 0x01 -#define DOC_ASICMODE_POWERDOWN 0x02 -#define DOC_ASICMODE_MDWREN 0x04 -#define DOC_ASICMODE_BDETCT_RESET 0x08 -#define DOC_ASICMODE_RSTIN_RESET 0x10 -#define DOC_ASICMODE_RAM_WE 0x20 - -/* good status values read after read/write/erase operations */ -#define DOCG4_PROGSTATUS_GOOD 0x51 -#define DOCG4_PROGSTATUS_GOOD_2 0xe0 - -/* - * On read operations (page and oob-only), the first byte read from I/O reg is a - * status. On error, it reads 0x73; otherwise, it reads either 0x71 (first read - * after reset only) or 0x51, so bit 1 is presumed to be an error indicator. - */ -#define DOCG4_READ_ERROR 0x02 /* bit 1 indicates read error */ - -/* anatomy of the device */ -#define DOCG4_CHIP_SIZE 0x8000000 -#define DOCG4_PAGE_SIZE 0x200 -#define DOCG4_PAGES_PER_BLOCK 0x200 -#define DOCG4_BLOCK_SIZE (DOCG4_PAGES_PER_BLOCK * DOCG4_PAGE_SIZE) -#define DOCG4_NUMBLOCKS (DOCG4_CHIP_SIZE / DOCG4_BLOCK_SIZE) -#define DOCG4_OOB_SIZE 0x10 -#define DOCG4_CHIP_SHIFT 27 /* log_2(DOCG4_CHIP_SIZE) */ -#define DOCG4_PAGE_SHIFT 9 /* log_2(DOCG4_PAGE_SIZE) */ -#define DOCG4_ERASE_SHIFT 18 /* log_2(DOCG4_BLOCK_SIZE) */ - -/* all but the last byte is included in ecc calculation */ -#define DOCG4_BCH_SIZE (DOCG4_PAGE_SIZE + DOCG4_OOB_SIZE - 1) - -#define DOCG4_USERDATA_LEN 520 /* 512 byte page plus 8 oob avail to user */ - -/* expected values from the ID registers */ -#define DOCG4_IDREG1_VALUE 0x0400 -#define DOCG4_IDREG2_VALUE 0xfbff - -/* primitive polynomial used to build the Galois field used by hw ecc gen */ -#define DOCG4_PRIMITIVE_POLY 0x4443 - -#define DOCG4_M 14 /* Galois field is of order 2^14 */ -#define DOCG4_T 4 /* BCH alg corrects up to 4 bit errors */ - -#define DOCG4_FACTORY_BBT_PAGE 16 /* page where read-only factory bbt lives */ -#define DOCG4_REDUNDANT_BBT_PAGE 24 /* page where redundant factory bbt lives */ - -/* - * Bytes 0, 1 are used as badblock marker. - * Bytes 2 - 6 are available to the user. - * Byte 7 is hamming ecc for first 7 oob bytes only. - * Bytes 8 - 14 are hw-generated ecc covering entire page + oob bytes 0 - 14. - * Byte 15 (the last) is used by the driver as a "page written" flag. - */ -static int docg4_ooblayout_ecc(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) -{ - if (section) - return -ERANGE; - - oobregion->offset = 7; - oobregion->length = 9; - - return 0; -} - -static int docg4_ooblayout_free(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) -{ - if (section) - return -ERANGE; - - oobregion->offset = 2; - oobregion->length = 5; - - return 0; -} - -static const struct mtd_ooblayout_ops docg4_ooblayout_ops = { - .ecc = docg4_ooblayout_ecc, - .free = docg4_ooblayout_free, -}; - -/* - * The device has a nop register which M-Sys claims is for the purpose of - * inserting precise delays. But beware; at least some operations fail if the - * nop writes are replaced with a generic delay! - */ -static inline void write_nop(void __iomem *docptr) -{ - writew(0, docptr + DOC_NOP); -} - -static void docg4_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) -{ - int i; - struct nand_chip *nand = mtd_to_nand(mtd); - uint16_t *p = (uint16_t *) buf; - len >>= 1; - - for (i = 0; i < len; i++) - p[i] = readw(nand->IO_ADDR_R); -} - -static void docg4_write_buf16(struct mtd_info *mtd, const uint8_t *buf, int len) -{ - int i; - struct nand_chip *nand = mtd_to_nand(mtd); - uint16_t *p = (uint16_t *) buf; - len >>= 1; - - for (i = 0; i < len; i++) - writew(p[i], nand->IO_ADDR_W); -} - -static int poll_status(struct docg4_priv *doc) -{ - /* - * Busy-wait for the FLASHREADY bit to be set in the FLASHCONTROL - * register. Operations known to take a long time (e.g., block erase) - * should sleep for a while before calling this. - */ - - uint16_t flash_status; - unsigned long timeo; - void __iomem *docptr = doc->virtadr; - - dev_dbg(doc->dev, "%s...\n", __func__); - - /* hardware quirk requires reading twice initially */ - flash_status = readw(docptr + DOC_FLASHCONTROL); - - timeo = jiffies + msecs_to_jiffies(200); /* generous timeout */ - do { - cpu_relax(); - flash_status = readb(docptr + DOC_FLASHCONTROL); - } while (!(flash_status & DOC_CTRL_FLASHREADY) && - time_before(jiffies, timeo)); - - if (unlikely(!(flash_status & DOC_CTRL_FLASHREADY))) { - dev_err(doc->dev, "%s: timed out!\n", __func__); - return NAND_STATUS_FAIL; - } - - return 0; -} - - -static int docg4_wait(struct mtd_info *mtd, struct nand_chip *nand) -{ - - struct docg4_priv *doc = nand_get_controller_data(nand); - int status = NAND_STATUS_WP; /* inverse logic?? */ - dev_dbg(doc->dev, "%s...\n", __func__); - - /* report any previously unreported error */ - if (doc->status) { - status |= doc->status; - doc->status = 0; - return status; - } - - status |= poll_status(doc); - return status; -} - -static void docg4_select_chip(struct mtd_info *mtd, int chip) -{ - /* - * Select among multiple cascaded chips ("floors"). Multiple floors are - * not yet supported, so the only valid non-negative value is 0. - */ - struct nand_chip *nand = mtd_to_nand(mtd); - struct docg4_priv *doc = nand_get_controller_data(nand); - void __iomem *docptr = doc->virtadr; - - dev_dbg(doc->dev, "%s: chip %d\n", __func__, chip); - - if (chip < 0) - return; /* deselected */ - - if (chip > 0) - dev_warn(doc->dev, "multiple floors currently unsupported\n"); - - writew(0, docptr + DOC_DEVICESELECT); -} - -static void reset(struct mtd_info *mtd) -{ - /* full device reset */ - - struct nand_chip *nand = mtd_to_nand(mtd); - struct docg4_priv *doc = nand_get_controller_data(nand); - void __iomem *docptr = doc->virtadr; - - writew(DOC_ASICMODE_RESET | DOC_ASICMODE_MDWREN, - docptr + DOC_ASICMODE); - writew(~(DOC_ASICMODE_RESET | DOC_ASICMODE_MDWREN), - docptr + DOC_ASICMODECONFIRM); - write_nop(docptr); - - writew(DOC_ASICMODE_NORMAL | DOC_ASICMODE_MDWREN, - docptr + DOC_ASICMODE); - writew(~(DOC_ASICMODE_NORMAL | DOC_ASICMODE_MDWREN), - docptr + DOC_ASICMODECONFIRM); - - writew(DOC_ECCCONF1_ECC_ENABLE, docptr + DOC_ECCCONF1); - - poll_status(doc); -} - -static void read_hw_ecc(void __iomem *docptr, uint8_t *ecc_buf) -{ - /* read the 7 hw-generated ecc bytes */ - - int i; - for (i = 0; i < 7; i++) { /* hw quirk; read twice */ - ecc_buf[i] = readb(docptr + DOC_BCH_SYNDROM(i)); - ecc_buf[i] = readb(docptr + DOC_BCH_SYNDROM(i)); - } -} - -static int correct_data(struct mtd_info *mtd, uint8_t *buf, int page) -{ - /* - * Called after a page read when hardware reports bitflips. - * Up to four bitflips can be corrected. - */ - - struct nand_chip *nand = mtd_to_nand(mtd); - struct docg4_priv *doc = nand_get_controller_data(nand); - void __iomem *docptr = doc->virtadr; - int i, numerrs, errpos[4]; - const uint8_t blank_read_hwecc[8] = { - 0xcf, 0x72, 0xfc, 0x1b, 0xa9, 0xc7, 0xb9, 0 }; - - read_hw_ecc(docptr, doc->ecc_buf); /* read 7 hw-generated ecc bytes */ - - /* check if read error is due to a blank page */ - if (!memcmp(doc->ecc_buf, blank_read_hwecc, 7)) - return 0; /* yes */ - - /* skip additional check of "written flag" if ignore_badblocks */ - if (ignore_badblocks == false) { - - /* - * If the hw ecc bytes are not those of a blank page, there's - * still a chance that the page is blank, but was read with - * errors. Check the "written flag" in last oob byte, which - * is set to zero when a page is written. If more than half - * the bits are set, assume a blank page. Unfortunately, the - * bit flips(s) are not reported in stats. - */ - - if (nand->oob_poi[15]) { - int bit, numsetbits = 0; - unsigned long written_flag = nand->oob_poi[15]; - for_each_set_bit(bit, &written_flag, 8) - numsetbits++; - if (numsetbits > 4) { /* assume blank */ - dev_warn(doc->dev, - "error(s) in blank page " - "at offset %08x\n", - page * DOCG4_PAGE_SIZE); - return 0; - } - } - } - - /* - * The hardware ecc unit produces oob_ecc ^ calc_ecc. The kernel's bch - * algorithm is used to decode this. However the hw operates on page - * data in a bit order that is the reverse of that of the bch alg, - * requiring that the bits be reversed on the result. Thanks to Ivan - * Djelic for his analysis! - */ - for (i = 0; i < 7; i++) - doc->ecc_buf[i] = bitrev8(doc->ecc_buf[i]); - - numerrs = decode_bch(doc->bch, NULL, DOCG4_USERDATA_LEN, NULL, - doc->ecc_buf, NULL, errpos); - - if (numerrs == -EBADMSG) { - dev_warn(doc->dev, "uncorrectable errors at offset %08x\n", - page * DOCG4_PAGE_SIZE); - return -EBADMSG; - } - - BUG_ON(numerrs < 0); /* -EINVAL, or anything other than -EBADMSG */ - - /* undo last step in BCH alg (modulo mirroring not needed) */ - for (i = 0; i < numerrs; i++) - errpos[i] = (errpos[i] & ~7)|(7-(errpos[i] & 7)); - - /* fix the errors */ - for (i = 0; i < numerrs; i++) { - - /* ignore if error within oob ecc bytes */ - if (errpos[i] > DOCG4_USERDATA_LEN * 8) - continue; - - /* if error within oob area preceeding ecc bytes... */ - if (errpos[i] > DOCG4_PAGE_SIZE * 8) - change_bit(errpos[i] - DOCG4_PAGE_SIZE * 8, - (unsigned long *)nand->oob_poi); - - else /* error in page data */ - change_bit(errpos[i], (unsigned long *)buf); - } - - dev_notice(doc->dev, "%d error(s) corrected at offset %08x\n", - numerrs, page * DOCG4_PAGE_SIZE); - - return numerrs; -} - -static uint8_t docg4_read_byte(struct mtd_info *mtd) -{ - struct nand_chip *nand = mtd_to_nand(mtd); - struct docg4_priv *doc = nand_get_controller_data(nand); - - dev_dbg(doc->dev, "%s\n", __func__); - - if (doc->last_command.command == NAND_CMD_STATUS) { - int status; - - /* - * Previous nand command was status request, so nand - * infrastructure code expects to read the status here. If an - * error occurred in a previous operation, report it. - */ - doc->last_command.command = 0; - - if (doc->status) { - status = doc->status; - doc->status = 0; - } - - /* why is NAND_STATUS_WP inverse logic?? */ - else - status = NAND_STATUS_WP | NAND_STATUS_READY; - - return status; - } - - dev_warn(doc->dev, "unexpected call to read_byte()\n"); - - return 0; -} - -static void write_addr(struct docg4_priv *doc, uint32_t docg4_addr) -{ - /* write the four address bytes packed in docg4_addr to the device */ - - void __iomem *docptr = doc->virtadr; - writeb(docg4_addr & 0xff, docptr + DOC_FLASHADDRESS); - docg4_addr >>= 8; - writeb(docg4_addr & 0xff, docptr + DOC_FLASHADDRESS); - docg4_addr >>= 8; - writeb(docg4_addr & 0xff, docptr + DOC_FLASHADDRESS); - docg4_addr >>= 8; - writeb(docg4_addr & 0xff, docptr + DOC_FLASHADDRESS); -} - -static int read_progstatus(struct docg4_priv *doc) -{ - /* - * This apparently checks the status of programming. Done after an - * erasure, and after page data is written. On error, the status is - * saved, to be later retrieved by the nand infrastructure code. - */ - void __iomem *docptr = doc->virtadr; - - /* status is read from the I/O reg */ - uint16_t status1 = readw(docptr + DOC_IOSPACE_DATA); - uint16_t status2 = readw(docptr + DOC_IOSPACE_DATA); - uint16_t status3 = readw(docptr + DOCG4_MYSTERY_REG); - - dev_dbg(doc->dev, "docg4: %s: %02x %02x %02x\n", - __func__, status1, status2, status3); - - if (status1 != DOCG4_PROGSTATUS_GOOD - || status2 != DOCG4_PROGSTATUS_GOOD_2 - || status3 != DOCG4_PROGSTATUS_GOOD_2) { - doc->status = NAND_STATUS_FAIL; - dev_warn(doc->dev, "read_progstatus failed: " - "%02x, %02x, %02x\n", status1, status2, status3); - return -EIO; - } - return 0; -} - -static int pageprog(struct mtd_info *mtd) -{ - /* - * Final step in writing a page. Writes the contents of its - * internal buffer out to the flash array, or some such. - */ - - struct nand_chip *nand = mtd_to_nand(mtd); - struct docg4_priv *doc = nand_get_controller_data(nand); - void __iomem *docptr = doc->virtadr; - int retval = 0; - - dev_dbg(doc->dev, "docg4: %s\n", __func__); - - writew(DOCG4_SEQ_PAGEPROG, docptr + DOC_FLASHSEQUENCE); - writew(DOC_CMD_PROG_CYCLE2, docptr + DOC_FLASHCOMMAND); - write_nop(docptr); - write_nop(docptr); - - /* Just busy-wait; usleep_range() slows things down noticeably. */ - poll_status(doc); - - writew(DOCG4_SEQ_FLUSH, docptr + DOC_FLASHSEQUENCE); - writew(DOCG4_CMD_FLUSH, docptr + DOC_FLASHCOMMAND); - writew(DOC_ECCCONF0_READ_MODE | 4, docptr + DOC_ECCCONF0); - write_nop(docptr); - write_nop(docptr); - write_nop(docptr); - write_nop(docptr); - write_nop(docptr); - - retval = read_progstatus(doc); - writew(0, docptr + DOC_DATAEND); - write_nop(docptr); - poll_status(doc); - write_nop(docptr); - - return retval; -} - -static void sequence_reset(struct mtd_info *mtd) -{ - /* common starting sequence for all operations */ - - struct nand_chip *nand = mtd_to_nand(mtd); - struct docg4_priv *doc = nand_get_controller_data(nand); - void __iomem *docptr = doc->virtadr; - - writew(DOC_CTRL_UNKNOWN | DOC_CTRL_CE, docptr + DOC_FLASHCONTROL); - writew(DOC_SEQ_RESET, docptr + DOC_FLASHSEQUENCE); - writew(DOC_CMD_RESET, docptr + DOC_FLASHCOMMAND); - write_nop(docptr); - write_nop(docptr); - poll_status(doc); - write_nop(docptr); -} - -static void read_page_prologue(struct mtd_info *mtd, uint32_t docg4_addr) -{ - /* first step in reading a page */ - - struct nand_chip *nand = mtd_to_nand(mtd); - struct docg4_priv *doc = nand_get_controller_data(nand); - void __iomem *docptr = doc->virtadr; - - dev_dbg(doc->dev, - "docg4: %s: g4 page %08x\n", __func__, docg4_addr); - - sequence_reset(mtd); - - writew(DOCG4_SEQ_PAGE_READ, docptr + DOC_FLASHSEQUENCE); - writew(DOCG4_CMD_PAGE_READ, docptr + DOC_FLASHCOMMAND); - write_nop(docptr); - - write_addr(doc, docg4_addr); - - write_nop(docptr); - writew(DOCG4_CMD_READ2, docptr + DOC_FLASHCOMMAND); - write_nop(docptr); - write_nop(docptr); - - poll_status(doc); -} - -static void write_page_prologue(struct mtd_info *mtd, uint32_t docg4_addr) -{ - /* first step in writing a page */ - - struct nand_chip *nand = mtd_to_nand(mtd); - struct docg4_priv *doc = nand_get_controller_data(nand); - void __iomem *docptr = doc->virtadr; - - dev_dbg(doc->dev, - "docg4: %s: g4 addr: %x\n", __func__, docg4_addr); - sequence_reset(mtd); - - if (unlikely(reliable_mode)) { - writew(DOCG4_SEQ_SETMODE, docptr + DOC_FLASHSEQUENCE); - writew(DOCG4_CMD_FAST_MODE, docptr + DOC_FLASHCOMMAND); - writew(DOC_CMD_RELIABLE_MODE, docptr + DOC_FLASHCOMMAND); - write_nop(docptr); - } - - writew(DOCG4_SEQ_PAGEWRITE, docptr + DOC_FLASHSEQUENCE); - writew(DOCG4_CMD_PAGEWRITE, docptr + DOC_FLASHCOMMAND); - write_nop(docptr); - write_addr(doc, docg4_addr); - write_nop(docptr); - write_nop(docptr); - poll_status(doc); -} - -static uint32_t mtd_to_docg4_address(int page, int column) -{ - /* - * Convert mtd address to format used by the device, 32 bit packed. - * - * Some notes on G4 addressing... The M-Sys documentation on this device - * claims that pages are 2K in length, and indeed, the format of the - * address used by the device reflects that. But within each page are - * four 512 byte "sub-pages", each with its own oob data that is - * read/written immediately after the 512 bytes of page data. This oob - * data contains the ecc bytes for the preceeding 512 bytes. - * - * Rather than tell the mtd nand infrastructure that page size is 2k, - * with four sub-pages each, we engage in a little subterfuge and tell - * the infrastructure code that pages are 512 bytes in size. This is - * done because during the course of reverse-engineering the device, I - * never observed an instance where an entire 2K "page" was read or - * written as a unit. Each "sub-page" is always addressed individually, - * its data read/written, and ecc handled before the next "sub-page" is - * addressed. - * - * This requires us to convert addresses passed by the mtd nand - * infrastructure code to those used by the device. - * - * The address that is written to the device consists of four bytes: the - * first two are the 2k page number, and the second is the index into - * the page. The index is in terms of 16-bit half-words and includes - * the preceeding oob data, so e.g., the index into the second - * "sub-page" is 0x108, and the full device address of the start of mtd - * page 0x201 is 0x00800108. - */ - int g4_page = page / 4; /* device's 2K page */ - int g4_index = (page % 4) * 0x108 + column/2; /* offset into page */ - return (g4_page << 16) | g4_index; /* pack */ -} - -static void docg4_command(struct mtd_info *mtd, unsigned command, int column, - int page_addr) -{ - /* handle standard nand commands */ - - struct nand_chip *nand = mtd_to_nand(mtd); - struct docg4_priv *doc = nand_get_controller_data(nand); - uint32_t g4_addr = mtd_to_docg4_address(page_addr, column); - - dev_dbg(doc->dev, "%s %x, page_addr=%x, column=%x\n", - __func__, command, page_addr, column); - - /* - * Save the command and its arguments. This enables emulation of - * standard flash devices, and also some optimizations. - */ - doc->last_command.command = command; - doc->last_command.column = column; - doc->last_command.page = page_addr; - - switch (command) { - - case NAND_CMD_RESET: - reset(mtd); - break; - - case NAND_CMD_READ0: - read_page_prologue(mtd, g4_addr); - break; - - case NAND_CMD_STATUS: - /* next call to read_byte() will expect a status */ - break; - - case NAND_CMD_SEQIN: - if (unlikely(reliable_mode)) { - uint16_t g4_page = g4_addr >> 16; - - /* writes to odd-numbered 2k pages are invalid */ - if (g4_page & 0x01) - dev_warn(doc->dev, - "invalid reliable mode address\n"); - } - - write_page_prologue(mtd, g4_addr); - - /* hack for deferred write of oob bytes */ - if (doc->oob_page == page_addr) - memcpy(nand->oob_poi, doc->oob_buf, 16); - break; - - case NAND_CMD_PAGEPROG: - pageprog(mtd); - break; - - /* we don't expect these, based on review of nand_base.c */ - case NAND_CMD_READOOB: - case NAND_CMD_READID: - case NAND_CMD_ERASE1: - case NAND_CMD_ERASE2: - dev_warn(doc->dev, "docg4_command: " - "unexpected nand command 0x%x\n", command); - break; - - } -} - -static int read_page(struct mtd_info *mtd, struct nand_chip *nand, - uint8_t *buf, int page, bool use_ecc) -{ - struct docg4_priv *doc = nand_get_controller_data(nand); - void __iomem *docptr = doc->virtadr; - uint16_t status, edc_err, *buf16; - int bits_corrected = 0; - - dev_dbg(doc->dev, "%s: page %08x\n", __func__, page); - - nand_read_page_op(nand, page, 0, NULL, 0); - - writew(DOC_ECCCONF0_READ_MODE | - DOC_ECCCONF0_ECC_ENABLE | - DOC_ECCCONF0_UNKNOWN | - DOCG4_BCH_SIZE, - docptr + DOC_ECCCONF0); - write_nop(docptr); - write_nop(docptr); - write_nop(docptr); - write_nop(docptr); - write_nop(docptr); - - /* the 1st byte from the I/O reg is a status; the rest is page data */ - status = readw(docptr + DOC_IOSPACE_DATA); - if (status & DOCG4_READ_ERROR) { - dev_err(doc->dev, - "docg4_read_page: bad status: 0x%02x\n", status); - writew(0, docptr + DOC_DATAEND); - return -EIO; - } - - dev_dbg(doc->dev, "%s: status = 0x%x\n", __func__, status); - - docg4_read_buf(mtd, buf, DOCG4_PAGE_SIZE); /* read the page data */ - - /* this device always reads oob after page data */ - /* first 14 oob bytes read from I/O reg */ - docg4_read_buf(mtd, nand->oob_poi, 14); - - /* last 2 read from another reg */ - buf16 = (uint16_t *)(nand->oob_poi + 14); - *buf16 = readw(docptr + DOCG4_MYSTERY_REG); - - write_nop(docptr); - - if (likely(use_ecc == true)) { - - /* read the register that tells us if bitflip(s) detected */ - edc_err = readw(docptr + DOC_ECCCONF1); - edc_err = readw(docptr + DOC_ECCCONF1); - dev_dbg(doc->dev, "%s: edc_err = 0x%02x\n", __func__, edc_err); - - /* If bitflips are reported, attempt to correct with ecc */ - if (edc_err & DOC_ECCCONF1_BCH_SYNDROM_ERR) { - bits_corrected = correct_data(mtd, buf, page); - if (bits_corrected == -EBADMSG) - mtd->ecc_stats.failed++; - else - mtd->ecc_stats.corrected += bits_corrected; - } - } - - writew(0, docptr + DOC_DATAEND); - if (bits_corrected == -EBADMSG) /* uncorrectable errors */ - return 0; - return bits_corrected; -} - - -static int docg4_read_page_raw(struct mtd_info *mtd, struct nand_chip *nand, - uint8_t *buf, int oob_required, int page) -{ - return read_page(mtd, nand, buf, page, false); -} - -static int docg4_read_page(struct mtd_info *mtd, struct nand_chip *nand, - uint8_t *buf, int oob_required, int page) -{ - return read_page(mtd, nand, buf, page, true); -} - -static int docg4_read_oob(struct mtd_info *mtd, struct nand_chip *nand, - int page) -{ - struct docg4_priv *doc = nand_get_controller_data(nand); - void __iomem *docptr = doc->virtadr; - uint16_t status; - - dev_dbg(doc->dev, "%s: page %x\n", __func__, page); - - nand_read_page_op(nand, page, nand->ecc.size, NULL, 0); - - writew(DOC_ECCCONF0_READ_MODE | DOCG4_OOB_SIZE, docptr + DOC_ECCCONF0); - write_nop(docptr); - write_nop(docptr); - write_nop(docptr); - write_nop(docptr); - write_nop(docptr); - - /* the 1st byte from the I/O reg is a status; the rest is oob data */ - status = readw(docptr + DOC_IOSPACE_DATA); - if (status & DOCG4_READ_ERROR) { - dev_warn(doc->dev, - "docg4_read_oob failed: status = 0x%02x\n", status); - return -EIO; - } - - dev_dbg(doc->dev, "%s: status = 0x%x\n", __func__, status); - - docg4_read_buf(mtd, nand->oob_poi, 16); - - write_nop(docptr); - write_nop(docptr); - write_nop(docptr); - writew(0, docptr + DOC_DATAEND); - write_nop(docptr); - - return 0; -} - -static int docg4_erase_block(struct mtd_info *mtd, int page) -{ - struct nand_chip *nand = mtd_to_nand(mtd); - struct docg4_priv *doc = nand_get_controller_data(nand); - void __iomem *docptr = doc->virtadr; - uint16_t g4_page; - int status; - - dev_dbg(doc->dev, "%s: page %04x\n", __func__, page); - - sequence_reset(mtd); - - writew(DOCG4_SEQ_BLOCKERASE, docptr + DOC_FLASHSEQUENCE); - writew(DOC_CMD_PROG_BLOCK_ADDR, docptr + DOC_FLASHCOMMAND); - write_nop(docptr); - - /* only 2 bytes of address are written to specify erase block */ - g4_page = (uint16_t)(page / 4); /* to g4's 2k page addressing */ - writeb(g4_page & 0xff, docptr + DOC_FLASHADDRESS); - g4_page >>= 8; - writeb(g4_page & 0xff, docptr + DOC_FLASHADDRESS); - write_nop(docptr); - - /* start the erasure */ - writew(DOC_CMD_ERASECYCLE2, docptr + DOC_FLASHCOMMAND); - write_nop(docptr); - write_nop(docptr); - - usleep_range(500, 1000); /* erasure is long; take a snooze */ - poll_status(doc); - writew(DOCG4_SEQ_FLUSH, docptr + DOC_FLASHSEQUENCE); - writew(DOCG4_CMD_FLUSH, docptr + DOC_FLASHCOMMAND); - writew(DOC_ECCCONF0_READ_MODE | 4, docptr + DOC_ECCCONF0); - write_nop(docptr); - write_nop(docptr); - write_nop(docptr); - write_nop(docptr); - write_nop(docptr); - - read_progstatus(doc); - - writew(0, docptr + DOC_DATAEND); - write_nop(docptr); - poll_status(doc); - write_nop(docptr); - - status = nand->waitfunc(mtd, nand); - if (status < 0) - return status; - - return status & NAND_STATUS_FAIL ? -EIO : 0; -} - -static int write_page(struct mtd_info *mtd, struct nand_chip *nand, - const uint8_t *buf, int page, bool use_ecc) -{ - struct docg4_priv *doc = nand_get_controller_data(nand); - void __iomem *docptr = doc->virtadr; - uint8_t ecc_buf[8]; - - dev_dbg(doc->dev, "%s...\n", __func__); - - nand_prog_page_begin_op(nand, page, 0, NULL, 0); - - writew(DOC_ECCCONF0_ECC_ENABLE | - DOC_ECCCONF0_UNKNOWN | - DOCG4_BCH_SIZE, - docptr + DOC_ECCCONF0); - write_nop(docptr); - - /* write the page data */ - docg4_write_buf16(mtd, buf, DOCG4_PAGE_SIZE); - - /* oob bytes 0 through 5 are written to I/O reg */ - docg4_write_buf16(mtd, nand->oob_poi, 6); - - /* oob byte 6 written to a separate reg */ - writew(nand->oob_poi[6], docptr + DOCG4_OOB_6_7); - - write_nop(docptr); - write_nop(docptr); - - /* write hw-generated ecc bytes to oob */ - if (likely(use_ecc == true)) { - /* oob byte 7 is hamming code */ - uint8_t hamming = readb(docptr + DOC_HAMMINGPARITY); - hamming = readb(docptr + DOC_HAMMINGPARITY); /* 2nd read */ - writew(hamming, docptr + DOCG4_OOB_6_7); - write_nop(docptr); - - /* read the 7 bch bytes from ecc regs */ - read_hw_ecc(docptr, ecc_buf); - ecc_buf[7] = 0; /* clear the "page written" flag */ - } - - /* write user-supplied bytes to oob */ - else { - writew(nand->oob_poi[7], docptr + DOCG4_OOB_6_7); - write_nop(docptr); - memcpy(ecc_buf, &nand->oob_poi[8], 8); - } - - docg4_write_buf16(mtd, ecc_buf, 8); - write_nop(docptr); - write_nop(docptr); - writew(0, docptr + DOC_DATAEND); - write_nop(docptr); - - return nand_prog_page_end_op(nand); -} - -static int docg4_write_page_raw(struct mtd_info *mtd, struct nand_chip *nand, - const uint8_t *buf, int oob_required, int page) -{ - return write_page(mtd, nand, buf, page, false); -} - -static int docg4_write_page(struct mtd_info *mtd, struct nand_chip *nand, - const uint8_t *buf, int oob_required, int page) -{ - return write_page(mtd, nand, buf, page, true); -} - -static int docg4_write_oob(struct mtd_info *mtd, struct nand_chip *nand, - int page) -{ - /* - * Writing oob-only is not really supported, because MLC nand must write - * oob bytes at the same time as page data. Nonetheless, we save the - * oob buffer contents here, and then write it along with the page data - * if the same page is subsequently written. This allows user space - * utilities that write the oob data prior to the page data to work - * (e.g., nandwrite). The disdvantage is that, if the intention was to - * write oob only, the operation is quietly ignored. Also, oob can get - * corrupted if two concurrent processes are running nandwrite. - */ - - /* note that bytes 7..14 are hw generated hamming/ecc and overwritten */ - struct docg4_priv *doc = nand_get_controller_data(nand); - doc->oob_page = page; - memcpy(doc->oob_buf, nand->oob_poi, 16); - return 0; -} - -static int __init read_factory_bbt(struct mtd_info *mtd) -{ - /* - * The device contains a read-only factory bad block table. Read it and - * update the memory-based bbt accordingly. - */ - - struct nand_chip *nand = mtd_to_nand(mtd); - struct docg4_priv *doc = nand_get_controller_data(nand); - uint32_t g4_addr = mtd_to_docg4_address(DOCG4_FACTORY_BBT_PAGE, 0); - uint8_t *buf; - int i, block; - __u32 eccfailed_stats = mtd->ecc_stats.failed; - - buf = kzalloc(DOCG4_PAGE_SIZE, GFP_KERNEL); - if (buf == NULL) - return -ENOMEM; - - read_page_prologue(mtd, g4_addr); - docg4_read_page(mtd, nand, buf, 0, DOCG4_FACTORY_BBT_PAGE); - - /* - * If no memory-based bbt was created, exit. This will happen if module - * parameter ignore_badblocks is set. Then why even call this function? - * For an unknown reason, block erase always fails if it's the first - * operation after device power-up. The above read ensures it never is. - * Ugly, I know. - */ - if (nand->bbt == NULL) /* no memory-based bbt */ - goto exit; - - if (mtd->ecc_stats.failed > eccfailed_stats) { - /* - * Whoops, an ecc failure ocurred reading the factory bbt. - * It is stored redundantly, so we get another chance. - */ - eccfailed_stats = mtd->ecc_stats.failed; - docg4_read_page(mtd, nand, buf, 0, DOCG4_REDUNDANT_BBT_PAGE); - if (mtd->ecc_stats.failed > eccfailed_stats) { - dev_warn(doc->dev, - "The factory bbt could not be read!\n"); - goto exit; - } - } - - /* - * Parse factory bbt and update memory-based bbt. Factory bbt format is - * simple: one bit per block, block numbers increase left to right (msb - * to lsb). Bit clear means bad block. - */ - for (i = block = 0; block < DOCG4_NUMBLOCKS; block += 8, i++) { - int bitnum; - unsigned long bits = ~buf[i]; - for_each_set_bit(bitnum, &bits, 8) { - int badblock = block + 7 - bitnum; - nand->bbt[badblock / 4] |= - 0x03 << ((badblock % 4) * 2); - mtd->ecc_stats.badblocks++; - dev_notice(doc->dev, "factory-marked bad block: %d\n", - badblock); - } - } - exit: - kfree(buf); - return 0; -} - -static int docg4_block_markbad(struct mtd_info *mtd, loff_t ofs) -{ - /* - * Mark a block as bad. Bad blocks are marked in the oob area of the - * first page of the block. The default scan_bbt() in the nand - * infrastructure code works fine for building the memory-based bbt - * during initialization, as does the nand infrastructure function that - * checks if a block is bad by reading the bbt. This function replaces - * the nand default because writes to oob-only are not supported. - */ - - int ret, i; - uint8_t *buf; - struct nand_chip *nand = mtd_to_nand(mtd); - struct docg4_priv *doc = nand_get_controller_data(nand); - struct nand_bbt_descr *bbtd = nand->badblock_pattern; - int page = (int)(ofs >> nand->page_shift); - uint32_t g4_addr = mtd_to_docg4_address(page, 0); - - dev_dbg(doc->dev, "%s: %08llx\n", __func__, ofs); - - if (unlikely(ofs & (DOCG4_BLOCK_SIZE - 1))) - dev_warn(doc->dev, "%s: ofs %llx not start of block!\n", - __func__, ofs); - - /* allocate blank buffer for page data */ - buf = kzalloc(DOCG4_PAGE_SIZE, GFP_KERNEL); - if (buf == NULL) - return -ENOMEM; - - /* write bit-wise negation of pattern to oob buffer */ - memset(nand->oob_poi, 0xff, mtd->oobsize); - for (i = 0; i < bbtd->len; i++) - nand->oob_poi[bbtd->offs + i] = ~bbtd->pattern[i]; - - /* write first page of block */ - write_page_prologue(mtd, g4_addr); - docg4_write_page(mtd, nand, buf, 1, page); - ret = pageprog(mtd); - - kfree(buf); - - return ret; -} - -static int docg4_block_neverbad(struct mtd_info *mtd, loff_t ofs) -{ - /* only called when module_param ignore_badblocks is set */ - return 0; -} - -static int docg4_suspend(struct platform_device *pdev, pm_message_t state) -{ - /* - * Put the device into "deep power-down" mode. Note that CE# must be - * deasserted for this to take effect. The xscale, e.g., can be - * configured to float this signal when the processor enters power-down, - * and a suitable pull-up ensures its deassertion. - */ - - int i; - uint8_t pwr_down; - struct docg4_priv *doc = platform_get_drvdata(pdev); - void __iomem *docptr = doc->virtadr; - - dev_dbg(doc->dev, "%s...\n", __func__); - - /* poll the register that tells us we're ready to go to sleep */ - for (i = 0; i < 10; i++) { - pwr_down = readb(docptr + DOC_POWERMODE); - if (pwr_down & DOC_POWERDOWN_READY) - break; - usleep_range(1000, 4000); - } - - if (pwr_down & DOC_POWERDOWN_READY) { - dev_err(doc->dev, "suspend failed; " - "timeout polling DOC_POWERDOWN_READY\n"); - return -EIO; - } - - writew(DOC_ASICMODE_POWERDOWN | DOC_ASICMODE_MDWREN, - docptr + DOC_ASICMODE); - writew(~(DOC_ASICMODE_POWERDOWN | DOC_ASICMODE_MDWREN), - docptr + DOC_ASICMODECONFIRM); - - write_nop(docptr); - - return 0; -} - -static int docg4_resume(struct platform_device *pdev) -{ - - /* - * Exit power-down. Twelve consecutive reads of the address below - * accomplishes this, assuming CE# has been asserted. - */ - - struct docg4_priv *doc = platform_get_drvdata(pdev); - void __iomem *docptr = doc->virtadr; - int i; - - dev_dbg(doc->dev, "%s...\n", __func__); - - for (i = 0; i < 12; i++) - readb(docptr + 0x1fff); - - return 0; -} - -static void init_mtd_structs(struct mtd_info *mtd) -{ - /* initialize mtd and nand data structures */ - - /* - * Note that some of the following initializations are not usually - * required within a nand driver because they are performed by the nand - * infrastructure code as part of nand_scan(). In this case they need - * to be initialized here because we skip call to nand_scan_ident() (the - * first half of nand_scan()). The call to nand_scan_ident() could be - * skipped because for this device the chip id is not read in the manner - * of a standard nand device. - */ - - struct nand_chip *nand = mtd_to_nand(mtd); - struct docg4_priv *doc = nand_get_controller_data(nand); - - mtd->size = DOCG4_CHIP_SIZE; - mtd->name = "Msys_Diskonchip_G4"; - mtd->writesize = DOCG4_PAGE_SIZE; - mtd->erasesize = DOCG4_BLOCK_SIZE; - mtd->oobsize = DOCG4_OOB_SIZE; - mtd_set_ooblayout(mtd, &docg4_ooblayout_ops); - nand->chipsize = DOCG4_CHIP_SIZE; - nand->chip_shift = DOCG4_CHIP_SHIFT; - nand->bbt_erase_shift = nand->phys_erase_shift = DOCG4_ERASE_SHIFT; - nand->chip_delay = 20; - nand->page_shift = DOCG4_PAGE_SHIFT; - nand->pagemask = 0x3ffff; - nand->badblockpos = NAND_LARGE_BADBLOCK_POS; - nand->badblockbits = 8; - nand->ecc.mode = NAND_ECC_HW_SYNDROME; - nand->ecc.size = DOCG4_PAGE_SIZE; - nand->ecc.prepad = 8; - nand->ecc.bytes = 8; - nand->ecc.strength = DOCG4_T; - nand->options = NAND_BUSWIDTH_16 | NAND_NO_SUBPAGE_WRITE; - nand->IO_ADDR_R = nand->IO_ADDR_W = doc->virtadr + DOC_IOSPACE_DATA; - nand->controller = &nand->dummy_controller; - nand_controller_init(nand->controller); - - /* methods */ - nand->cmdfunc = docg4_command; - nand->waitfunc = docg4_wait; - nand->select_chip = docg4_select_chip; - nand->read_byte = docg4_read_byte; - nand->block_markbad = docg4_block_markbad; - nand->read_buf = docg4_read_buf; - nand->write_buf = docg4_write_buf16; - nand->erase = docg4_erase_block; - nand->set_features = nand_get_set_features_notsupp; - nand->get_features = nand_get_set_features_notsupp; - nand->ecc.read_page = docg4_read_page; - nand->ecc.write_page = docg4_write_page; - nand->ecc.read_page_raw = docg4_read_page_raw; - nand->ecc.write_page_raw = docg4_write_page_raw; - nand->ecc.read_oob = docg4_read_oob; - nand->ecc.write_oob = docg4_write_oob; - - /* - * The way the nand infrastructure code is written, a memory-based bbt - * is not created if NAND_SKIP_BBTSCAN is set. With no memory bbt, - * nand->block_bad() is used. So when ignoring bad blocks, we skip the - * scan and define a dummy block_bad() which always returns 0. - */ - if (ignore_badblocks) { - nand->options |= NAND_SKIP_BBTSCAN; - nand->block_bad = docg4_block_neverbad; - } - -} - -static int read_id_reg(struct mtd_info *mtd) -{ - struct nand_chip *nand = mtd_to_nand(mtd); - struct docg4_priv *doc = nand_get_controller_data(nand); - void __iomem *docptr = doc->virtadr; - uint16_t id1, id2; - - /* check for presence of g4 chip by reading id registers */ - id1 = readw(docptr + DOC_CHIPID); - id1 = readw(docptr + DOCG4_MYSTERY_REG); - id2 = readw(docptr + DOC_CHIPID_INV); - id2 = readw(docptr + DOCG4_MYSTERY_REG); - - if (id1 == DOCG4_IDREG1_VALUE && id2 == DOCG4_IDREG2_VALUE) { - dev_info(doc->dev, - "NAND device: 128MiB Diskonchip G4 detected\n"); - return 0; - } - - return -ENODEV; -} - -static char const *part_probes[] = { "cmdlinepart", "saftlpart", NULL }; - -static int docg4_attach_chip(struct nand_chip *chip) -{ - struct mtd_info *mtd = nand_to_mtd(chip); - struct docg4_priv *doc = (struct docg4_priv *)(chip + 1); - int ret; - - init_mtd_structs(mtd); - - /* Initialize kernel BCH algorithm */ - doc->bch = init_bch(DOCG4_M, DOCG4_T, DOCG4_PRIMITIVE_POLY); - if (!doc->bch) - return -EINVAL; - - reset(mtd); - - ret = read_id_reg(mtd); - if (ret) - free_bch(doc->bch); - - return ret; -} - -static void docg4_detach_chip(struct nand_chip *chip) -{ - struct docg4_priv *doc = (struct docg4_priv *)(chip + 1); - - free_bch(doc->bch); -} - -static const struct nand_controller_ops docg4_controller_ops = { - .attach_chip = docg4_attach_chip, - .detach_chip = docg4_detach_chip, -}; - -static int __init probe_docg4(struct platform_device *pdev) -{ - struct mtd_info *mtd; - struct nand_chip *nand; - void __iomem *virtadr; - struct docg4_priv *doc; - int len, retval; - struct resource *r; - struct device *dev = &pdev->dev; - - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (r == NULL) { - dev_err(dev, "no io memory resource defined!\n"); - return -ENODEV; - } - - virtadr = ioremap(r->start, resource_size(r)); - if (!virtadr) { - dev_err(dev, "Diskonchip ioremap failed: %pR\n", r); - return -EIO; - } - - len = sizeof(struct nand_chip) + sizeof(struct docg4_priv); - nand = kzalloc(len, GFP_KERNEL); - if (nand == NULL) { - retval = -ENOMEM; - goto unmap; - } - - mtd = nand_to_mtd(nand); - doc = (struct docg4_priv *) (nand + 1); - nand_set_controller_data(nand, doc); - mtd->dev.parent = &pdev->dev; - doc->virtadr = virtadr; - doc->dev = dev; - platform_set_drvdata(pdev, doc); - - /* - * Running nand_scan() with maxchips == 0 will skip nand_scan_ident(), - * which is a specific operation with this driver and done in the - * ->attach_chip callback. - */ - nand->dummy_controller.ops = &docg4_controller_ops; - retval = nand_scan(mtd, 0); - if (retval) - goto free_nand; - - retval = read_factory_bbt(mtd); - if (retval) - goto cleanup_nand; - - retval = mtd_device_parse_register(mtd, part_probes, NULL, NULL, 0); - if (retval) - goto cleanup_nand; - - doc->mtd = mtd; - - return 0; - -cleanup_nand: - nand_cleanup(nand); -free_nand: - kfree(nand); -unmap: - iounmap(virtadr); - - return retval; -} - -static int __exit cleanup_docg4(struct platform_device *pdev) -{ - struct docg4_priv *doc = platform_get_drvdata(pdev); - nand_release(doc->mtd); - kfree(mtd_to_nand(doc->mtd)); - iounmap(doc->virtadr); - return 0; -} - -static struct platform_driver docg4_driver = { - .driver = { - .name = "docg4", - }, - .suspend = docg4_suspend, - .resume = docg4_resume, - .remove = __exit_p(cleanup_docg4), -}; - -module_platform_driver_probe(docg4_driver, probe_docg4); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Mike Dunn"); -MODULE_DESCRIPTION("M-Systems DiskOnChip G4 device driver"); diff --git a/drivers/mtd/nand/raw/fsl_elbc_nand.c b/drivers/mtd/nand/raw/fsl_elbc_nand.c index 55f449b711fd..d6ed697fcfe6 100644 --- a/drivers/mtd/nand/raw/fsl_elbc_nand.c +++ b/drivers/mtd/nand/raw/fsl_elbc_nand.c @@ -317,10 +317,10 @@ static void fsl_elbc_do_read(struct nand_chip *chip, int oob) } /* cmdfunc send commands to the FCM */ -static void fsl_elbc_cmdfunc(struct mtd_info *mtd, unsigned int command, +static void fsl_elbc_cmdfunc(struct nand_chip *chip, unsigned int command, int column, int page_addr) { - struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(chip); struct fsl_elbc_mtd *priv = nand_get_controller_data(chip); struct fsl_lbc_ctrl *ctrl = priv->ctrl; struct fsl_elbc_fcm_ctrl *elbc_fcm_ctrl = ctrl->nand; @@ -533,7 +533,7 @@ static void fsl_elbc_cmdfunc(struct mtd_info *mtd, unsigned int command, } } -static void fsl_elbc_select_chip(struct mtd_info *mtd, int chip) +static void fsl_elbc_select_chip(struct nand_chip *chip, int cs) { /* The hardware does not seem to support multiple * chips per bank. @@ -543,9 +543,9 @@ static void fsl_elbc_select_chip(struct mtd_info *mtd, int chip) /* * Write buf to the FCM Controller Data Buffer */ -static void fsl_elbc_write_buf(struct mtd_info *mtd, const u8 *buf, int len) +static void fsl_elbc_write_buf(struct nand_chip *chip, const u8 *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(chip); struct fsl_elbc_mtd *priv = nand_get_controller_data(chip); struct fsl_elbc_fcm_ctrl *elbc_fcm_ctrl = priv->ctrl->nand; unsigned int bufsize = mtd->writesize + mtd->oobsize; @@ -581,9 +581,8 @@ static void fsl_elbc_write_buf(struct mtd_info *mtd, const u8 *buf, int len) * read a byte from either the FCM hardware buffer if it has any data left * otherwise issue a command to read a single byte. */ -static u8 fsl_elbc_read_byte(struct mtd_info *mtd) +static u8 fsl_elbc_read_byte(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct fsl_elbc_mtd *priv = nand_get_controller_data(chip); struct fsl_elbc_fcm_ctrl *elbc_fcm_ctrl = priv->ctrl->nand; @@ -598,9 +597,8 @@ static u8 fsl_elbc_read_byte(struct mtd_info *mtd) /* * Read from the FCM Controller Data Buffer */ -static void fsl_elbc_read_buf(struct mtd_info *mtd, u8 *buf, int len) +static void fsl_elbc_read_buf(struct nand_chip *chip, u8 *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct fsl_elbc_mtd *priv = nand_get_controller_data(chip); struct fsl_elbc_fcm_ctrl *elbc_fcm_ctrl = priv->ctrl->nand; int avail; @@ -623,7 +621,7 @@ static void fsl_elbc_read_buf(struct mtd_info *mtd, u8 *buf, int len) /* This function is called after Program and Erase Operations to * check for success or failure. */ -static int fsl_elbc_wait(struct mtd_info *mtd, struct nand_chip *chip) +static int fsl_elbc_wait(struct nand_chip *chip) { struct fsl_elbc_mtd *priv = nand_get_controller_data(chip); struct fsl_elbc_fcm_ctrl *elbc_fcm_ctrl = priv->ctrl->nand; @@ -660,8 +658,8 @@ static int fsl_elbc_attach_chip(struct nand_chip *chip) chip->chipsize); dev_dbg(priv->dev, "fsl_elbc_init: nand->pagemask = %8x\n", chip->pagemask); - dev_dbg(priv->dev, "fsl_elbc_init: nand->chip_delay = %d\n", - chip->chip_delay); + dev_dbg(priv->dev, "fsl_elbc_init: nand->legacy.chip_delay = %d\n", + chip->legacy.chip_delay); dev_dbg(priv->dev, "fsl_elbc_init: nand->badblockpos = %d\n", chip->badblockpos); dev_dbg(priv->dev, "fsl_elbc_init: nand->chip_shift = %d\n", @@ -710,18 +708,19 @@ static const struct nand_controller_ops fsl_elbc_controller_ops = { .attach_chip = fsl_elbc_attach_chip, }; -static int fsl_elbc_read_page(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int fsl_elbc_read_page(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct fsl_elbc_mtd *priv = nand_get_controller_data(chip); struct fsl_lbc_ctrl *ctrl = priv->ctrl; struct fsl_elbc_fcm_ctrl *elbc_fcm_ctrl = ctrl->nand; nand_read_page_op(chip, page, 0, buf, mtd->writesize); if (oob_required) - fsl_elbc_read_buf(mtd, chip->oob_poi, mtd->oobsize); + fsl_elbc_read_buf(chip, chip->oob_poi, mtd->oobsize); - if (fsl_elbc_wait(mtd, chip) & NAND_STATUS_FAIL) + if (fsl_elbc_wait(chip) & NAND_STATUS_FAIL) mtd->ecc_stats.failed++; return elbc_fcm_ctrl->max_bitflips; @@ -730,11 +729,13 @@ static int fsl_elbc_read_page(struct mtd_info *mtd, struct nand_chip *chip, /* ECC will be calculated automatically, and errors will be detected in * waitfunc. */ -static int fsl_elbc_write_page(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, int page) +static int fsl_elbc_write_page(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize); - fsl_elbc_write_buf(mtd, chip->oob_poi, mtd->oobsize); + fsl_elbc_write_buf(chip, chip->oob_poi, mtd->oobsize); return nand_prog_page_end_op(chip); } @@ -742,13 +743,15 @@ static int fsl_elbc_write_page(struct mtd_info *mtd, struct nand_chip *chip, /* ECC will be calculated automatically, and errors will be detected in * waitfunc. */ -static int fsl_elbc_write_subpage(struct mtd_info *mtd, struct nand_chip *chip, - uint32_t offset, uint32_t data_len, - const uint8_t *buf, int oob_required, int page) +static int fsl_elbc_write_subpage(struct nand_chip *chip, uint32_t offset, + uint32_t data_len, const uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + nand_prog_page_begin_op(chip, page, 0, NULL, 0); - fsl_elbc_write_buf(mtd, buf, mtd->writesize); - fsl_elbc_write_buf(mtd, chip->oob_poi, mtd->oobsize); + fsl_elbc_write_buf(chip, buf, mtd->writesize); + fsl_elbc_write_buf(chip, chip->oob_poi, mtd->oobsize); return nand_prog_page_end_op(chip); } @@ -773,14 +776,14 @@ static int fsl_elbc_chip_init(struct fsl_elbc_mtd *priv) /* fill in nand_chip structure */ /* set up function call table */ - chip->read_byte = fsl_elbc_read_byte; - chip->write_buf = fsl_elbc_write_buf; - chip->read_buf = fsl_elbc_read_buf; + chip->legacy.read_byte = fsl_elbc_read_byte; + chip->legacy.write_buf = fsl_elbc_write_buf; + chip->legacy.read_buf = fsl_elbc_read_buf; chip->select_chip = fsl_elbc_select_chip; - chip->cmdfunc = fsl_elbc_cmdfunc; - chip->waitfunc = fsl_elbc_wait; - chip->set_features = nand_get_set_features_notsupp; - chip->get_features = nand_get_set_features_notsupp; + chip->legacy.cmdfunc = fsl_elbc_cmdfunc; + chip->legacy.waitfunc = fsl_elbc_wait; + chip->legacy.set_features = nand_get_set_features_notsupp; + chip->legacy.get_features = nand_get_set_features_notsupp; chip->bbt_td = &bbt_main_descr; chip->bbt_md = &bbt_mirror_descr; @@ -915,7 +918,7 @@ static int fsl_elbc_nand_probe(struct platform_device *pdev) goto err; priv->chip.controller->ops = &fsl_elbc_controller_ops; - ret = nand_scan(mtd, 1); + ret = nand_scan(&priv->chip, 1); if (ret) goto err; @@ -942,9 +945,8 @@ static int fsl_elbc_nand_remove(struct platform_device *pdev) { struct fsl_elbc_fcm_ctrl *elbc_fcm_ctrl = fsl_lbc_ctrl_dev->nand; struct fsl_elbc_mtd *priv = dev_get_drvdata(&pdev->dev); - struct mtd_info *mtd = nand_to_mtd(&priv->chip); - nand_release(mtd); + nand_release(&priv->chip); fsl_elbc_chip_remove(priv); mutex_lock(&fsl_elbc_nand_mutex); diff --git a/drivers/mtd/nand/raw/fsl_ifc_nand.c b/drivers/mtd/nand/raw/fsl_ifc_nand.c index 24f59d0066af..6f4afc44381a 100644 --- a/drivers/mtd/nand/raw/fsl_ifc_nand.c +++ b/drivers/mtd/nand/raw/fsl_ifc_nand.c @@ -30,6 +30,7 @@ #include <linux/mtd/partitions.h> #include <linux/mtd/nand_ecc.h> #include <linux/fsl_ifc.h> +#include <linux/iopoll.h> #define ERR_BYTE 0xFF /* Value returned for read bytes when read failed */ @@ -300,9 +301,9 @@ static void fsl_ifc_do_read(struct nand_chip *chip, } /* cmdfunc send commands to the IFC NAND Machine */ -static void fsl_ifc_cmdfunc(struct mtd_info *mtd, unsigned int command, - int column, int page_addr) { - struct nand_chip *chip = mtd_to_nand(mtd); +static void fsl_ifc_cmdfunc(struct nand_chip *chip, unsigned int command, + int column, int page_addr) { + struct mtd_info *mtd = nand_to_mtd(chip); struct fsl_ifc_mtd *priv = nand_get_controller_data(chip); struct fsl_ifc_ctrl *ctrl = priv->ctrl; struct fsl_ifc_runtime __iomem *ifc = ctrl->rregs; @@ -508,7 +509,7 @@ static void fsl_ifc_cmdfunc(struct mtd_info *mtd, unsigned int command, } } -static void fsl_ifc_select_chip(struct mtd_info *mtd, int chip) +static void fsl_ifc_select_chip(struct nand_chip *chip, int cs) { /* The hardware does not seem to support multiple * chips per bank. @@ -518,9 +519,9 @@ static void fsl_ifc_select_chip(struct mtd_info *mtd, int chip) /* * Write buf to the IFC NAND Controller Data Buffer */ -static void fsl_ifc_write_buf(struct mtd_info *mtd, const u8 *buf, int len) +static void fsl_ifc_write_buf(struct nand_chip *chip, const u8 *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(chip); struct fsl_ifc_mtd *priv = nand_get_controller_data(chip); unsigned int bufsize = mtd->writesize + mtd->oobsize; @@ -544,9 +545,8 @@ static void fsl_ifc_write_buf(struct mtd_info *mtd, const u8 *buf, int len) * Read a byte from either the IFC hardware buffer * read function for 8-bit buswidth */ -static uint8_t fsl_ifc_read_byte(struct mtd_info *mtd) +static uint8_t fsl_ifc_read_byte(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct fsl_ifc_mtd *priv = nand_get_controller_data(chip); unsigned int offset; @@ -567,9 +567,8 @@ static uint8_t fsl_ifc_read_byte(struct mtd_info *mtd) * Read two bytes from the IFC hardware buffer * read function for 16-bit buswith */ -static uint8_t fsl_ifc_read_byte16(struct mtd_info *mtd) +static uint8_t fsl_ifc_read_byte16(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct fsl_ifc_mtd *priv = nand_get_controller_data(chip); uint16_t data; @@ -590,9 +589,8 @@ static uint8_t fsl_ifc_read_byte16(struct mtd_info *mtd) /* * Read from the IFC Controller Data Buffer */ -static void fsl_ifc_read_buf(struct mtd_info *mtd, u8 *buf, int len) +static void fsl_ifc_read_buf(struct nand_chip *chip, u8 *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct fsl_ifc_mtd *priv = nand_get_controller_data(chip); int avail; @@ -616,8 +614,9 @@ static void fsl_ifc_read_buf(struct mtd_info *mtd, u8 *buf, int len) * This function is called after Program and Erase Operations to * check for success or failure. */ -static int fsl_ifc_wait(struct mtd_info *mtd, struct nand_chip *chip) +static int fsl_ifc_wait(struct nand_chip *chip) { + struct mtd_info *mtd = nand_to_mtd(chip); struct fsl_ifc_mtd *priv = nand_get_controller_data(chip); struct fsl_ifc_ctrl *ctrl = priv->ctrl; struct fsl_ifc_runtime __iomem *ifc = ctrl->rregs; @@ -678,20 +677,21 @@ static int check_erased_page(struct nand_chip *chip, u8 *buf) return bitflips; } -static int fsl_ifc_read_page(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int fsl_ifc_read_page(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct fsl_ifc_mtd *priv = nand_get_controller_data(chip); struct fsl_ifc_ctrl *ctrl = priv->ctrl; struct fsl_ifc_nand_ctrl *nctrl = ifc_nand_ctrl; nand_read_page_op(chip, page, 0, buf, mtd->writesize); if (oob_required) - fsl_ifc_read_buf(mtd, chip->oob_poi, mtd->oobsize); + fsl_ifc_read_buf(chip, chip->oob_poi, mtd->oobsize); if (ctrl->nand_stat & IFC_NAND_EVTER_STAT_ECCER) { if (!oob_required) - fsl_ifc_read_buf(mtd, chip->oob_poi, mtd->oobsize); + fsl_ifc_read_buf(chip, chip->oob_poi, mtd->oobsize); return check_erased_page(chip, buf); } @@ -705,11 +705,13 @@ static int fsl_ifc_read_page(struct mtd_info *mtd, struct nand_chip *chip, /* ECC will be calculated automatically, and errors will be detected in * waitfunc. */ -static int fsl_ifc_write_page(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, int page) +static int fsl_ifc_write_page(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize); - fsl_ifc_write_buf(mtd, chip->oob_poi, mtd->oobsize); + fsl_ifc_write_buf(chip, chip->oob_poi, mtd->oobsize); return nand_prog_page_end_op(chip); } @@ -725,8 +727,8 @@ static int fsl_ifc_attach_chip(struct nand_chip *chip) chip->chipsize); dev_dbg(priv->dev, "%s: nand->pagemask = %8x\n", __func__, chip->pagemask); - dev_dbg(priv->dev, "%s: nand->chip_delay = %d\n", __func__, - chip->chip_delay); + dev_dbg(priv->dev, "%s: nand->legacy.chip_delay = %d\n", __func__, + chip->legacy.chip_delay); dev_dbg(priv->dev, "%s: nand->badblockpos = %d\n", __func__, chip->badblockpos); dev_dbg(priv->dev, "%s: nand->chip_shift = %d\n", __func__, @@ -761,7 +763,7 @@ static const struct nand_controller_ops fsl_ifc_controller_ops = { .attach_chip = fsl_ifc_attach_chip, }; -static void fsl_ifc_sram_init(struct fsl_ifc_mtd *priv) +static int fsl_ifc_sram_init(struct fsl_ifc_mtd *priv) { struct fsl_ifc_ctrl *ctrl = priv->ctrl; struct fsl_ifc_runtime __iomem *ifc_runtime = ctrl->rregs; @@ -769,6 +771,27 @@ static void fsl_ifc_sram_init(struct fsl_ifc_mtd *priv) uint32_t csor = 0, csor_8k = 0, csor_ext = 0; uint32_t cs = priv->bank; + if (ctrl->version < FSL_IFC_VERSION_1_1_0) + return 0; + + if (ctrl->version > FSL_IFC_VERSION_1_1_0) { + u32 ncfgr, status; + int ret; + + /* Trigger auto initialization */ + ncfgr = ifc_in32(&ifc_runtime->ifc_nand.ncfgr); + ifc_out32(ncfgr | IFC_NAND_NCFGR_SRAM_INIT_EN, &ifc_runtime->ifc_nand.ncfgr); + + /* Wait until done */ + ret = readx_poll_timeout(ifc_in32, &ifc_runtime->ifc_nand.ncfgr, + status, !(status & IFC_NAND_NCFGR_SRAM_INIT_EN), + 10, IFC_TIMEOUT_MSECS * 1000); + if (ret) + dev_err(priv->dev, "Failed to initialize SRAM!\n"); + + return ret; + } + /* Save CSOR and CSOR_ext */ csor = ifc_in32(&ifc_global->csor_cs[cs].csor); csor_ext = ifc_in32(&ifc_global->csor_cs[cs].csor_ext); @@ -805,12 +828,16 @@ static void fsl_ifc_sram_init(struct fsl_ifc_mtd *priv) wait_event_timeout(ctrl->nand_wait, ctrl->nand_stat, msecs_to_jiffies(IFC_TIMEOUT_MSECS)); - if (ctrl->nand_stat != IFC_NAND_EVTER_STAT_OPC) + if (ctrl->nand_stat != IFC_NAND_EVTER_STAT_OPC) { pr_err("fsl-ifc: Failed to Initialise SRAM\n"); + return -ETIMEDOUT; + } /* Restore CSOR and CSOR_ext */ ifc_out32(csor, &ifc_global->csor_cs[cs].csor); ifc_out32(csor_ext, &ifc_global->csor_cs[cs].csor_ext); + + return 0; } static int fsl_ifc_chip_init(struct fsl_ifc_mtd *priv) @@ -821,6 +848,7 @@ static int fsl_ifc_chip_init(struct fsl_ifc_mtd *priv) struct nand_chip *chip = &priv->chip; struct mtd_info *mtd = nand_to_mtd(&priv->chip); u32 csor; + int ret; /* Fill in fsl_ifc_mtd structure */ mtd->dev.parent = priv->dev; @@ -830,17 +858,17 @@ static int fsl_ifc_chip_init(struct fsl_ifc_mtd *priv) /* set up function call table */ if ((ifc_in32(&ifc_global->cspr_cs[priv->bank].cspr)) & CSPR_PORT_SIZE_16) - chip->read_byte = fsl_ifc_read_byte16; + chip->legacy.read_byte = fsl_ifc_read_byte16; else - chip->read_byte = fsl_ifc_read_byte; + chip->legacy.read_byte = fsl_ifc_read_byte; - chip->write_buf = fsl_ifc_write_buf; - chip->read_buf = fsl_ifc_read_buf; + chip->legacy.write_buf = fsl_ifc_write_buf; + chip->legacy.read_buf = fsl_ifc_read_buf; chip->select_chip = fsl_ifc_select_chip; - chip->cmdfunc = fsl_ifc_cmdfunc; - chip->waitfunc = fsl_ifc_wait; - chip->set_features = nand_get_set_features_notsupp; - chip->get_features = nand_get_set_features_notsupp; + chip->legacy.cmdfunc = fsl_ifc_cmdfunc; + chip->legacy.waitfunc = fsl_ifc_wait; + chip->legacy.set_features = nand_get_set_features_notsupp; + chip->legacy.get_features = nand_get_set_features_notsupp; chip->bbt_td = &bbt_main_descr; chip->bbt_md = &bbt_mirror_descr; @@ -853,10 +881,10 @@ static int fsl_ifc_chip_init(struct fsl_ifc_mtd *priv) if (ifc_in32(&ifc_global->cspr_cs[priv->bank].cspr) & CSPR_PORT_SIZE_16) { - chip->read_byte = fsl_ifc_read_byte16; + chip->legacy.read_byte = fsl_ifc_read_byte16; chip->options |= NAND_BUSWIDTH_16; } else { - chip->read_byte = fsl_ifc_read_byte; + chip->legacy.read_byte = fsl_ifc_read_byte; } chip->controller = &ifc_nand_ctrl->controller; @@ -914,8 +942,9 @@ static int fsl_ifc_chip_init(struct fsl_ifc_mtd *priv) chip->ecc.algo = NAND_ECC_HAMMING; } - if (ctrl->version >= FSL_IFC_VERSION_1_1_0) - fsl_ifc_sram_init(priv); + ret = fsl_ifc_sram_init(priv); + if (ret) + return ret; /* * As IFC version 2.0.0 has 16KB of internal SRAM as compared to older @@ -1051,7 +1080,7 @@ static int fsl_ifc_nand_probe(struct platform_device *dev) goto err; priv->chip.controller->ops = &fsl_ifc_controller_ops; - ret = nand_scan(mtd, 1); + ret = nand_scan(&priv->chip, 1); if (ret) goto err; @@ -1077,9 +1106,8 @@ err: static int fsl_ifc_nand_remove(struct platform_device *dev) { struct fsl_ifc_mtd *priv = dev_get_drvdata(&dev->dev); - struct mtd_info *mtd = nand_to_mtd(&priv->chip); - nand_release(mtd); + nand_release(&priv->chip); fsl_ifc_chip_remove(priv); mutex_lock(&fsl_ifc_nand_mutex); diff --git a/drivers/mtd/nand/raw/fsl_upm.c b/drivers/mtd/nand/raw/fsl_upm.c index a88e2cf66e0f..673c5a0c9345 100644 --- a/drivers/mtd/nand/raw/fsl_upm.c +++ b/drivers/mtd/nand/raw/fsl_upm.c @@ -52,9 +52,9 @@ static inline struct fsl_upm_nand *to_fsl_upm_nand(struct mtd_info *mtdinfo) chip); } -static int fun_chip_ready(struct mtd_info *mtd) +static int fun_chip_ready(struct nand_chip *chip) { - struct fsl_upm_nand *fun = to_fsl_upm_nand(mtd); + struct fsl_upm_nand *fun = to_fsl_upm_nand(nand_to_mtd(chip)); if (gpio_get_value(fun->rnb_gpio[fun->mchip_number])) return 1; @@ -69,7 +69,7 @@ static void fun_wait_rnb(struct fsl_upm_nand *fun) struct mtd_info *mtd = nand_to_mtd(&fun->chip); int cnt = 1000000; - while (--cnt && !fun_chip_ready(mtd)) + while (--cnt && !fun_chip_ready(&fun->chip)) cpu_relax(); if (!cnt) dev_err(fun->dev, "tired waiting for RNB\n"); @@ -78,10 +78,9 @@ static void fun_wait_rnb(struct fsl_upm_nand *fun) } } -static void fun_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl) +static void fun_cmd_ctrl(struct nand_chip *chip, int cmd, unsigned int ctrl) { - struct nand_chip *chip = mtd_to_nand(mtd); - struct fsl_upm_nand *fun = to_fsl_upm_nand(mtd); + struct fsl_upm_nand *fun = to_fsl_upm_nand(nand_to_mtd(chip)); u32 mar; if (!(ctrl & fun->last_ctrl)) { @@ -102,51 +101,50 @@ static void fun_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl) mar = (cmd << (32 - fun->upm.width)) | fun->mchip_offsets[fun->mchip_number]; - fsl_upm_run_pattern(&fun->upm, chip->IO_ADDR_R, mar); + fsl_upm_run_pattern(&fun->upm, chip->legacy.IO_ADDR_R, mar); if (fun->wait_flags & FSL_UPM_WAIT_RUN_PATTERN) fun_wait_rnb(fun); } -static void fun_select_chip(struct mtd_info *mtd, int mchip_nr) +static void fun_select_chip(struct nand_chip *chip, int mchip_nr) { - struct nand_chip *chip = mtd_to_nand(mtd); - struct fsl_upm_nand *fun = to_fsl_upm_nand(mtd); + struct fsl_upm_nand *fun = to_fsl_upm_nand(nand_to_mtd(chip)); if (mchip_nr == -1) { - chip->cmd_ctrl(mtd, NAND_CMD_NONE, 0 | NAND_CTRL_CHANGE); + chip->legacy.cmd_ctrl(chip, NAND_CMD_NONE, 0 | NAND_CTRL_CHANGE); } else if (mchip_nr >= 0 && mchip_nr < NAND_MAX_CHIPS) { fun->mchip_number = mchip_nr; - chip->IO_ADDR_R = fun->io_base + fun->mchip_offsets[mchip_nr]; - chip->IO_ADDR_W = chip->IO_ADDR_R; + chip->legacy.IO_ADDR_R = fun->io_base + fun->mchip_offsets[mchip_nr]; + chip->legacy.IO_ADDR_W = chip->legacy.IO_ADDR_R; } else { BUG(); } } -static uint8_t fun_read_byte(struct mtd_info *mtd) +static uint8_t fun_read_byte(struct nand_chip *chip) { - struct fsl_upm_nand *fun = to_fsl_upm_nand(mtd); + struct fsl_upm_nand *fun = to_fsl_upm_nand(nand_to_mtd(chip)); - return in_8(fun->chip.IO_ADDR_R); + return in_8(fun->chip.legacy.IO_ADDR_R); } -static void fun_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) +static void fun_read_buf(struct nand_chip *chip, uint8_t *buf, int len) { - struct fsl_upm_nand *fun = to_fsl_upm_nand(mtd); + struct fsl_upm_nand *fun = to_fsl_upm_nand(nand_to_mtd(chip)); int i; for (i = 0; i < len; i++) - buf[i] = in_8(fun->chip.IO_ADDR_R); + buf[i] = in_8(fun->chip.legacy.IO_ADDR_R); } -static void fun_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len) +static void fun_write_buf(struct nand_chip *chip, const uint8_t *buf, int len) { - struct fsl_upm_nand *fun = to_fsl_upm_nand(mtd); + struct fsl_upm_nand *fun = to_fsl_upm_nand(nand_to_mtd(chip)); int i; for (i = 0; i < len; i++) { - out_8(fun->chip.IO_ADDR_W, buf[i]); + out_8(fun->chip.legacy.IO_ADDR_W, buf[i]); if (fun->wait_flags & FSL_UPM_WAIT_WRITE_BYTE) fun_wait_rnb(fun); } @@ -162,20 +160,20 @@ static int fun_chip_init(struct fsl_upm_nand *fun, int ret; struct device_node *flash_np; - fun->chip.IO_ADDR_R = fun->io_base; - fun->chip.IO_ADDR_W = fun->io_base; - fun->chip.cmd_ctrl = fun_cmd_ctrl; - fun->chip.chip_delay = fun->chip_delay; - fun->chip.read_byte = fun_read_byte; - fun->chip.read_buf = fun_read_buf; - fun->chip.write_buf = fun_write_buf; + fun->chip.legacy.IO_ADDR_R = fun->io_base; + fun->chip.legacy.IO_ADDR_W = fun->io_base; + fun->chip.legacy.cmd_ctrl = fun_cmd_ctrl; + fun->chip.legacy.chip_delay = fun->chip_delay; + fun->chip.legacy.read_byte = fun_read_byte; + fun->chip.legacy.read_buf = fun_read_buf; + fun->chip.legacy.write_buf = fun_write_buf; fun->chip.ecc.mode = NAND_ECC_SOFT; fun->chip.ecc.algo = NAND_ECC_HAMMING; if (fun->mchip_count > 1) fun->chip.select_chip = fun_select_chip; if (fun->rnb_gpio[0] >= 0) - fun->chip.dev_ready = fun_chip_ready; + fun->chip.legacy.dev_ready = fun_chip_ready; mtd->dev.parent = fun->dev; @@ -184,14 +182,14 @@ static int fun_chip_init(struct fsl_upm_nand *fun, return -ENODEV; nand_set_flash_node(&fun->chip, flash_np); - mtd->name = kasprintf(GFP_KERNEL, "0x%llx.%s", (u64)io_res->start, - flash_np->name); + mtd->name = kasprintf(GFP_KERNEL, "0x%llx.%pOFn", (u64)io_res->start, + flash_np); if (!mtd->name) { ret = -ENOMEM; goto err; } - ret = nand_scan(mtd, fun->mchip_count); + ret = nand_scan(&fun->chip, fun->mchip_count); if (ret) goto err; @@ -326,7 +324,7 @@ static int fun_remove(struct platform_device *ofdev) struct mtd_info *mtd = nand_to_mtd(&fun->chip); int i; - nand_release(mtd); + nand_release(&fun->chip); kfree(mtd->name); for (i = 0; i < fun->mchip_count; i++) { diff --git a/drivers/mtd/nand/raw/fsmc_nand.c b/drivers/mtd/nand/raw/fsmc_nand.c index f418236fa020..70ac8d875218 100644 --- a/drivers/mtd/nand/raw/fsmc_nand.c +++ b/drivers/mtd/nand/raw/fsmc_nand.c @@ -340,10 +340,9 @@ static int fsmc_calc_timings(struct fsmc_nand_data *host, return 0; } -static int fsmc_setup_data_interface(struct mtd_info *mtd, int csline, +static int fsmc_setup_data_interface(struct nand_chip *nand, int csline, const struct nand_data_interface *conf) { - struct nand_chip *nand = mtd_to_nand(mtd); struct fsmc_nand_data *host = nand_get_controller_data(nand); struct fsmc_nand_timings tims; const struct nand_sdr_timings *sdrt; @@ -368,9 +367,9 @@ static int fsmc_setup_data_interface(struct mtd_info *mtd, int csline, /* * fsmc_enable_hwecc - Enables Hardware ECC through FSMC registers */ -static void fsmc_enable_hwecc(struct mtd_info *mtd, int mode) +static void fsmc_enable_hwecc(struct nand_chip *chip, int mode) { - struct fsmc_nand_data *host = mtd_to_fsmc(mtd); + struct fsmc_nand_data *host = mtd_to_fsmc(nand_to_mtd(chip)); writel_relaxed(readl(host->regs_va + FSMC_PC) & ~FSMC_ECCPLEN_256, host->regs_va + FSMC_PC); @@ -385,10 +384,10 @@ static void fsmc_enable_hwecc(struct mtd_info *mtd, int mode) * FSMC. ECC is 13 bytes for 512 bytes of data (supports error correction up to * max of 8-bits) */ -static int fsmc_read_hwecc_ecc4(struct mtd_info *mtd, const uint8_t *data, +static int fsmc_read_hwecc_ecc4(struct nand_chip *chip, const uint8_t *data, uint8_t *ecc) { - struct fsmc_nand_data *host = mtd_to_fsmc(mtd); + struct fsmc_nand_data *host = mtd_to_fsmc(nand_to_mtd(chip)); uint32_t ecc_tmp; unsigned long deadline = jiffies + FSMC_BUSY_WAIT_TIMEOUT; @@ -433,10 +432,10 @@ static int fsmc_read_hwecc_ecc4(struct mtd_info *mtd, const uint8_t *data, * FSMC. ECC is 3 bytes for 512 bytes of data (supports error correction up to * max of 1-bit) */ -static int fsmc_read_hwecc_ecc1(struct mtd_info *mtd, const uint8_t *data, +static int fsmc_read_hwecc_ecc1(struct nand_chip *chip, const uint8_t *data, uint8_t *ecc) { - struct fsmc_nand_data *host = mtd_to_fsmc(mtd); + struct fsmc_nand_data *host = mtd_to_fsmc(nand_to_mtd(chip)); uint32_t ecc_tmp; ecc_tmp = readl_relaxed(host->regs_va + ECC1); @@ -610,9 +609,9 @@ static void fsmc_write_buf_dma(struct mtd_info *mtd, const uint8_t *buf, } /* fsmc_select_chip - assert or deassert nCE */ -static void fsmc_select_chip(struct mtd_info *mtd, int chipnr) +static void fsmc_select_chip(struct nand_chip *chip, int chipnr) { - struct fsmc_nand_data *host = mtd_to_fsmc(mtd); + struct fsmc_nand_data *host = mtd_to_fsmc(nand_to_mtd(chip)); u32 pc; /* Support only one CS */ @@ -707,7 +706,6 @@ static int fsmc_exec_op(struct nand_chip *chip, const struct nand_operation *op, /* * fsmc_read_page_hwecc - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: buffer to store read data * @oob_required: caller expects OOB data read to chip->oob_poi @@ -719,9 +717,10 @@ static int fsmc_exec_op(struct nand_chip *chip, const struct nand_operation *op, * After this read, fsmc hardware generates and reports error data bits(up to a * max of 8 bits) */ -static int fsmc_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int fsmc_read_page_hwecc(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int i, j, s, stat, eccsize = chip->ecc.size; int eccbytes = chip->ecc.bytes; int eccsteps = chip->ecc.steps; @@ -740,7 +739,7 @@ static int fsmc_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, for (i = 0, s = 0; s < eccsteps; s++, i += eccbytes, p += eccsize) { nand_read_page_op(chip, page, s * eccsize, NULL, 0); - chip->ecc.hwctl(mtd, NAND_ECC_READ); + chip->ecc.hwctl(chip, NAND_ECC_READ); nand_read_data_op(chip, p, eccsize, false); for (j = 0; j < eccbytes;) { @@ -767,9 +766,9 @@ static int fsmc_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, } memcpy(&ecc_code[i], oob, chip->ecc.bytes); - chip->ecc.calculate(mtd, p, &ecc_calc[i]); + chip->ecc.calculate(chip, p, &ecc_calc[i]); - stat = chip->ecc.correct(mtd, p, &ecc_code[i], &ecc_calc[i]); + stat = chip->ecc.correct(chip, p, &ecc_code[i], &ecc_calc[i]); if (stat < 0) { mtd->ecc_stats.failed++; } else { @@ -791,11 +790,10 @@ static int fsmc_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, * calc_ecc is a 104 bit information containing maximum of 8 error * offset informations of 13 bits each in 512 bytes of read data. */ -static int fsmc_bch8_correct_data(struct mtd_info *mtd, uint8_t *dat, - uint8_t *read_ecc, uint8_t *calc_ecc) +static int fsmc_bch8_correct_data(struct nand_chip *chip, uint8_t *dat, + uint8_t *read_ecc, uint8_t *calc_ecc) { - struct nand_chip *chip = mtd_to_nand(mtd); - struct fsmc_nand_data *host = mtd_to_fsmc(mtd); + struct fsmc_nand_data *host = mtd_to_fsmc(nand_to_mtd(chip)); uint32_t err_idx[8]; uint32_t num_err, i; uint32_t ecc1, ecc2, ecc3, ecc4; @@ -951,6 +949,7 @@ static int fsmc_nand_attach_chip(struct nand_chip *nand) nand->ecc.correct = nand_correct_data; nand->ecc.bytes = 3; nand->ecc.strength = 1; + nand->ecc.options |= NAND_ECC_SOFT_HAMMING_SM_ORDER; break; case NAND_ECC_SOFT: @@ -1082,7 +1081,6 @@ static int __init fsmc_nand_probe(struct platform_device *pdev) mtd->dev.parent = &pdev->dev; nand->exec_op = fsmc_exec_op; nand->select_chip = fsmc_select_chip; - nand->chip_delay = 30; /* * Setup default ECC mode. nand_dt_init() called from nand_scan_ident() @@ -1125,7 +1123,7 @@ static int __init fsmc_nand_probe(struct platform_device *pdev) * Scan to find existence of the device */ nand->dummy_controller.ops = &fsmc_nand_controller_ops; - ret = nand_scan(mtd, 1); + ret = nand_scan(nand, 1); if (ret) goto release_dma_write_chan; @@ -1161,7 +1159,7 @@ static int fsmc_nand_remove(struct platform_device *pdev) struct fsmc_nand_data *host = platform_get_drvdata(pdev); if (host) { - nand_release(nand_to_mtd(&host->nand)); + nand_release(&host->nand); if (host->mode == USE_DMA_ACCESS) { dma_release_channel(host->write_dma_chan); diff --git a/drivers/mtd/nand/raw/gpio.c b/drivers/mtd/nand/raw/gpio.c index 2780af26d9ab..a6c9a824a7d4 100644 --- a/drivers/mtd/nand/raw/gpio.c +++ b/drivers/mtd/nand/raw/gpio.c @@ -73,9 +73,10 @@ static void gpio_nand_dosync(struct gpiomtd *gpiomtd) static inline void gpio_nand_dosync(struct gpiomtd *gpiomtd) {} #endif -static void gpio_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl) +static void gpio_nand_cmd_ctrl(struct nand_chip *chip, int cmd, + unsigned int ctrl) { - struct gpiomtd *gpiomtd = gpio_nand_getpriv(mtd); + struct gpiomtd *gpiomtd = gpio_nand_getpriv(nand_to_mtd(chip)); gpio_nand_dosync(gpiomtd); @@ -89,13 +90,13 @@ static void gpio_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl) if (cmd == NAND_CMD_NONE) return; - writeb(cmd, gpiomtd->nand_chip.IO_ADDR_W); + writeb(cmd, gpiomtd->nand_chip.legacy.IO_ADDR_W); gpio_nand_dosync(gpiomtd); } -static int gpio_nand_devready(struct mtd_info *mtd) +static int gpio_nand_devready(struct nand_chip *chip) { - struct gpiomtd *gpiomtd = gpio_nand_getpriv(mtd); + struct gpiomtd *gpiomtd = gpio_nand_getpriv(nand_to_mtd(chip)); return gpiod_get_value(gpiomtd->rdy); } @@ -194,7 +195,7 @@ static int gpio_nand_remove(struct platform_device *pdev) { struct gpiomtd *gpiomtd = platform_get_drvdata(pdev); - nand_release(nand_to_mtd(&gpiomtd->nand_chip)); + nand_release(&gpiomtd->nand_chip); /* Enable write protection and disable the chip */ if (gpiomtd->nwp && !IS_ERR(gpiomtd->nwp)) @@ -224,9 +225,9 @@ static int gpio_nand_probe(struct platform_device *pdev) chip = &gpiomtd->nand_chip; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - chip->IO_ADDR_R = devm_ioremap_resource(dev, res); - if (IS_ERR(chip->IO_ADDR_R)) - return PTR_ERR(chip->IO_ADDR_R); + chip->legacy.IO_ADDR_R = devm_ioremap_resource(dev, res); + if (IS_ERR(chip->legacy.IO_ADDR_R)) + return PTR_ERR(chip->legacy.IO_ADDR_R); res = gpio_nand_get_io_sync(pdev); if (res) { @@ -270,15 +271,15 @@ static int gpio_nand_probe(struct platform_device *pdev) } /* Using RDY pin */ if (gpiomtd->rdy) - chip->dev_ready = gpio_nand_devready; + chip->legacy.dev_ready = gpio_nand_devready; nand_set_flash_node(chip, pdev->dev.of_node); - chip->IO_ADDR_W = chip->IO_ADDR_R; + chip->legacy.IO_ADDR_W = chip->legacy.IO_ADDR_R; chip->ecc.mode = NAND_ECC_SOFT; chip->ecc.algo = NAND_ECC_HAMMING; chip->options = gpiomtd->plat.options; - chip->chip_delay = gpiomtd->plat.chip_delay; - chip->cmd_ctrl = gpio_nand_cmd_ctrl; + chip->legacy.chip_delay = gpiomtd->plat.chip_delay; + chip->legacy.cmd_ctrl = gpio_nand_cmd_ctrl; mtd = nand_to_mtd(chip); mtd->dev.parent = dev; @@ -289,7 +290,7 @@ static int gpio_nand_probe(struct platform_device *pdev) if (gpiomtd->nwp && !IS_ERR(gpiomtd->nwp)) gpiod_direction_output(gpiomtd->nwp, 1); - ret = nand_scan(mtd, 1); + ret = nand_scan(chip, 1); if (ret) goto err_wp; diff --git a/drivers/mtd/nand/raw/gpmi-nand/gpmi-lib.c b/drivers/mtd/nand/raw/gpmi-nand/gpmi-lib.c index 88ea2203e263..bd4cfac6b5aa 100644 --- a/drivers/mtd/nand/raw/gpmi-nand/gpmi-lib.c +++ b/drivers/mtd/nand/raw/gpmi-nand/gpmi-lib.c @@ -471,10 +471,9 @@ void gpmi_nfc_apply_timings(struct gpmi_nand_data *this) udelay(dll_wait_time_us); } -int gpmi_setup_data_interface(struct mtd_info *mtd, int chipnr, +int gpmi_setup_data_interface(struct nand_chip *chip, int chipnr, const struct nand_data_interface *conf) { - struct nand_chip *chip = mtd_to_nand(mtd); struct gpmi_nand_data *this = nand_get_controller_data(chip); const struct nand_sdr_timings *sdr; diff --git a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c index 1c1ebbc82824..94c2b7525c85 100644 --- a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c +++ b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c @@ -783,9 +783,8 @@ error_alloc: return -ENOMEM; } -static void gpmi_cmd_ctrl(struct mtd_info *mtd, int data, unsigned int ctrl) +static void gpmi_cmd_ctrl(struct nand_chip *chip, int data, unsigned int ctrl) { - struct nand_chip *chip = mtd_to_nand(mtd); struct gpmi_nand_data *this = nand_get_controller_data(chip); int ret; @@ -817,17 +816,15 @@ static void gpmi_cmd_ctrl(struct mtd_info *mtd, int data, unsigned int ctrl) this->command_length = 0; } -static int gpmi_dev_ready(struct mtd_info *mtd) +static int gpmi_dev_ready(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct gpmi_nand_data *this = nand_get_controller_data(chip); return gpmi_is_ready(this, this->current_chip); } -static void gpmi_select_chip(struct mtd_info *mtd, int chipnr) +static void gpmi_select_chip(struct nand_chip *chip, int chipnr) { - struct nand_chip *chip = mtd_to_nand(mtd); struct gpmi_nand_data *this = nand_get_controller_data(chip); int ret; @@ -859,9 +856,8 @@ static void gpmi_select_chip(struct mtd_info *mtd, int chipnr) this->current_chip = chipnr; } -static void gpmi_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) +static void gpmi_read_buf(struct nand_chip *chip, uint8_t *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct gpmi_nand_data *this = nand_get_controller_data(chip); dev_dbg(this->dev, "len is %d\n", len); @@ -869,9 +865,8 @@ static void gpmi_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) gpmi_read_data(this, buf, len); } -static void gpmi_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len) +static void gpmi_write_buf(struct nand_chip *chip, const uint8_t *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct gpmi_nand_data *this = nand_get_controller_data(chip); dev_dbg(this->dev, "len is %d\n", len); @@ -879,13 +874,12 @@ static void gpmi_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len) gpmi_send_data(this, buf, len); } -static uint8_t gpmi_read_byte(struct mtd_info *mtd) +static uint8_t gpmi_read_byte(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct gpmi_nand_data *this = nand_get_controller_data(chip); uint8_t *buf = this->data_buffer_dma; - gpmi_read_buf(mtd, buf, 1); + gpmi_read_buf(chip, buf, 1); return buf[0]; } @@ -1085,8 +1079,8 @@ static int gpmi_ecc_read_page_data(struct nand_chip *chip, return max_bitflips; } -static int gpmi_ecc_read_page(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int gpmi_ecc_read_page(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { nand_read_page_op(chip, page, 0, NULL, 0); @@ -1094,8 +1088,8 @@ static int gpmi_ecc_read_page(struct mtd_info *mtd, struct nand_chip *chip, } /* Fake a virtual small page for the subpage read */ -static int gpmi_ecc_read_subpage(struct mtd_info *mtd, struct nand_chip *chip, - uint32_t offs, uint32_t len, uint8_t *buf, int page) +static int gpmi_ecc_read_subpage(struct nand_chip *chip, uint32_t offs, + uint32_t len, uint8_t *buf, int page) { struct gpmi_nand_data *this = nand_get_controller_data(chip); void __iomem *bch_regs = this->resources.bch_regs; @@ -1130,7 +1124,7 @@ static int gpmi_ecc_read_subpage(struct mtd_info *mtd, struct nand_chip *chip, dev_dbg(this->dev, "page:%d, first:%d, last:%d, marker at:%d\n", page, first, last, marker_pos); - return gpmi_ecc_read_page(mtd, chip, buf, 0, page); + return gpmi_ecc_read_page(chip, buf, 0, page); } } @@ -1182,9 +1176,10 @@ static int gpmi_ecc_read_subpage(struct mtd_info *mtd, struct nand_chip *chip, return max_bitflips; } -static int gpmi_ecc_write_page(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, int page) +static int gpmi_ecc_write_page(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct gpmi_nand_data *this = nand_get_controller_data(chip); struct bch_geometry *nfc_geo = &this->bch_geometry; const void *payload_virt; @@ -1324,9 +1319,9 @@ exit_auxiliary: * ECC-based or raw view of the page is implicit in which function it calls * (there is a similar pair of ECC-based/raw functions for writing). */ -static int gpmi_ecc_read_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int gpmi_ecc_read_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct gpmi_nand_data *this = nand_get_controller_data(chip); dev_dbg(this->dev, "page number is %d\n", page); @@ -1335,7 +1330,7 @@ static int gpmi_ecc_read_oob(struct mtd_info *mtd, struct nand_chip *chip, /* Read out the conventional OOB. */ nand_read_page_op(chip, page, mtd->writesize, NULL, 0); - chip->read_buf(mtd, chip->oob_poi, mtd->oobsize); + chip->legacy.read_buf(chip, chip->oob_poi, mtd->oobsize); /* * Now, we want to make sure the block mark is correct. In the @@ -1345,15 +1340,15 @@ static int gpmi_ecc_read_oob(struct mtd_info *mtd, struct nand_chip *chip, if (GPMI_IS_MX23(this)) { /* Read the block mark into the first byte of the OOB buffer. */ nand_read_page_op(chip, page, 0, NULL, 0); - chip->oob_poi[0] = chip->read_byte(mtd); + chip->oob_poi[0] = chip->legacy.read_byte(chip); } return 0; } -static int -gpmi_ecc_write_oob(struct mtd_info *mtd, struct nand_chip *chip, int page) +static int gpmi_ecc_write_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct mtd_oob_region of = { }; /* Do we have available oob area? */ @@ -1380,10 +1375,10 @@ gpmi_ecc_write_oob(struct mtd_info *mtd, struct nand_chip *chip, int page) * See set_geometry_by_ecc_info inline comments to have a full description * of the layout used by the GPMI controller. */ -static int gpmi_ecc_read_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, uint8_t *buf, +static int gpmi_ecc_read_page_raw(struct nand_chip *chip, uint8_t *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct gpmi_nand_data *this = nand_get_controller_data(chip); struct bch_geometry *nfc_geo = &this->bch_geometry; int eccsize = nfc_geo->ecc_chunk_size; @@ -1464,11 +1459,10 @@ static int gpmi_ecc_read_page_raw(struct mtd_info *mtd, * See set_geometry_by_ecc_info inline comments to have a full description * of the layout used by the GPMI controller. */ -static int gpmi_ecc_write_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, - const uint8_t *buf, +static int gpmi_ecc_write_page_raw(struct nand_chip *chip, const uint8_t *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct gpmi_nand_data *this = nand_get_controller_data(chip); struct bch_geometry *nfc_geo = &this->bch_geometry; int eccsize = nfc_geo->ecc_chunk_size; @@ -1536,28 +1530,26 @@ static int gpmi_ecc_write_page_raw(struct mtd_info *mtd, mtd->writesize + mtd->oobsize); } -static int gpmi_ecc_read_oob_raw(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int gpmi_ecc_read_oob_raw(struct nand_chip *chip, int page) { - return gpmi_ecc_read_page_raw(mtd, chip, NULL, 1, page); + return gpmi_ecc_read_page_raw(chip, NULL, 1, page); } -static int gpmi_ecc_write_oob_raw(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int gpmi_ecc_write_oob_raw(struct nand_chip *chip, int page) { - return gpmi_ecc_write_page_raw(mtd, chip, NULL, 1, page); + return gpmi_ecc_write_page_raw(chip, NULL, 1, page); } -static int gpmi_block_markbad(struct mtd_info *mtd, loff_t ofs) +static int gpmi_block_markbad(struct nand_chip *chip, loff_t ofs) { - struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(chip); struct gpmi_nand_data *this = nand_get_controller_data(chip); int ret = 0; uint8_t *block_mark; int column, page, chipnr; chipnr = (int)(ofs >> chip->chip_shift); - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, chipnr); column = !GPMI_IS_MX23(this) ? mtd->writesize : 0; @@ -1570,7 +1562,7 @@ static int gpmi_block_markbad(struct mtd_info *mtd, loff_t ofs) ret = nand_prog_page_op(chip, page, column, block_mark, 1); - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); return ret; } @@ -1607,7 +1599,6 @@ static int mx23_check_transcription_stamp(struct gpmi_nand_data *this) struct boot_rom_geometry *rom_geo = &this->rom_geometry; struct device *dev = this->dev; struct nand_chip *chip = &this->nand; - struct mtd_info *mtd = nand_to_mtd(chip); unsigned int search_area_size_in_strides; unsigned int stride; unsigned int page; @@ -1619,7 +1610,7 @@ static int mx23_check_transcription_stamp(struct gpmi_nand_data *this) search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent; saved_chip_number = this->current_chip; - chip->select_chip(mtd, 0); + chip->select_chip(chip, 0); /* * Loop through the first search area, looking for the NCB fingerprint. @@ -1637,7 +1628,7 @@ static int mx23_check_transcription_stamp(struct gpmi_nand_data *this) * and starts in the 12th byte of the page. */ nand_read_page_op(chip, page, 12, NULL, 0); - chip->read_buf(mtd, buffer, strlen(fingerprint)); + chip->legacy.read_buf(chip, buffer, strlen(fingerprint)); /* Look for the fingerprint. */ if (!memcmp(buffer, fingerprint, strlen(fingerprint))) { @@ -1647,7 +1638,7 @@ static int mx23_check_transcription_stamp(struct gpmi_nand_data *this) } - chip->select_chip(mtd, saved_chip_number); + chip->select_chip(chip, saved_chip_number); if (found_an_ncb_fingerprint) dev_dbg(dev, "\tFound a fingerprint\n"); @@ -1690,7 +1681,7 @@ static int mx23_write_transcription_stamp(struct gpmi_nand_data *this) /* Select chip 0. */ saved_chip_number = this->current_chip; - chip->select_chip(mtd, 0); + chip->select_chip(chip, 0); /* Loop over blocks in the first search area, erasing them. */ dev_dbg(dev, "Erasing the search area...\n"); @@ -1716,13 +1707,13 @@ static int mx23_write_transcription_stamp(struct gpmi_nand_data *this) /* Write the first page of the current stride. */ dev_dbg(dev, "Writing an NCB fingerprint in page 0x%x\n", page); - status = chip->ecc.write_page_raw(mtd, chip, buffer, 0, page); + status = chip->ecc.write_page_raw(chip, buffer, 0, page); if (status) dev_err(dev, "[%s] Write failed.\n", __func__); } /* Deselect chip 0. */ - chip->select_chip(mtd, saved_chip_number); + chip->select_chip(chip, saved_chip_number); return 0; } @@ -1771,10 +1762,10 @@ static int mx23_boot_init(struct gpmi_nand_data *this) byte = block << chip->phys_erase_shift; /* Send the command to read the conventional block mark. */ - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, chipnr); nand_read_page_op(chip, page, mtd->writesize, NULL, 0); - block_mark = chip->read_byte(mtd); - chip->select_chip(mtd, -1); + block_mark = chip->legacy.read_byte(chip); + chip->select_chip(chip, -1); /* * Check if the block is marked bad. If so, we need to mark it @@ -1783,7 +1774,7 @@ static int mx23_boot_init(struct gpmi_nand_data *this) */ if (block_mark != 0xff) { dev_dbg(dev, "Transcribing mark in block %u\n", block); - ret = chip->block_markbad(mtd, byte); + ret = chip->legacy.block_markbad(chip, byte); if (ret) dev_err(dev, "Failed to mark block bad with ret %d\n", @@ -1911,13 +1902,13 @@ static int gpmi_nand_init(struct gpmi_nand_data *this) nand_set_flash_node(chip, this->pdev->dev.of_node); chip->select_chip = gpmi_select_chip; chip->setup_data_interface = gpmi_setup_data_interface; - chip->cmd_ctrl = gpmi_cmd_ctrl; - chip->dev_ready = gpmi_dev_ready; - chip->read_byte = gpmi_read_byte; - chip->read_buf = gpmi_read_buf; - chip->write_buf = gpmi_write_buf; + chip->legacy.cmd_ctrl = gpmi_cmd_ctrl; + chip->legacy.dev_ready = gpmi_dev_ready; + chip->legacy.read_byte = gpmi_read_byte; + chip->legacy.read_buf = gpmi_read_buf; + chip->legacy.write_buf = gpmi_write_buf; chip->badblock_pattern = &gpmi_bbt_descr; - chip->block_markbad = gpmi_block_markbad; + chip->legacy.block_markbad = gpmi_block_markbad; chip->options |= NAND_NO_SUBPAGE_WRITE; /* Set up swap_block_mark, must be set before the gpmi_set_geometry() */ @@ -1934,7 +1925,7 @@ static int gpmi_nand_init(struct gpmi_nand_data *this) goto err_out; chip->dummy_controller.ops = &gpmi_nand_controller_ops; - ret = nand_scan(mtd, GPMI_IS_MX6(this) ? 2 : 1); + ret = nand_scan(chip, GPMI_IS_MX6(this) ? 2 : 1); if (ret) goto err_out; @@ -2026,7 +2017,7 @@ static int gpmi_nand_remove(struct platform_device *pdev) { struct gpmi_nand_data *this = platform_get_drvdata(pdev); - nand_release(nand_to_mtd(&this->nand)); + nand_release(&this->nand); gpmi_free_dma_buffer(this); release_resources(this); return 0; diff --git a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.h b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.h index 69cd0cbde4f2..d0b79bac2728 100644 --- a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.h +++ b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.h @@ -178,7 +178,7 @@ int gpmi_is_ready(struct gpmi_nand_data *, unsigned chip); int gpmi_send_command(struct gpmi_nand_data *); int gpmi_enable_clk(struct gpmi_nand_data *this); int gpmi_disable_clk(struct gpmi_nand_data *this); -int gpmi_setup_data_interface(struct mtd_info *mtd, int chipnr, +int gpmi_setup_data_interface(struct nand_chip *chip, int chipnr, const struct nand_data_interface *conf); void gpmi_nfc_apply_timings(struct gpmi_nand_data *this); int gpmi_read_data(struct gpmi_nand_data *, void *buf, int len); diff --git a/drivers/mtd/nand/raw/hisi504_nand.c b/drivers/mtd/nand/raw/hisi504_nand.c index 950dc7789296..f043938ee36b 100644 --- a/drivers/mtd/nand/raw/hisi504_nand.c +++ b/drivers/mtd/nand/raw/hisi504_nand.c @@ -353,9 +353,8 @@ static int hisi_nfc_send_cmd_reset(struct hinfc_host *host, int chipselect) return 0; } -static void hisi_nfc_select_chip(struct mtd_info *mtd, int chipselect) +static void hisi_nfc_select_chip(struct nand_chip *chip, int chipselect) { - struct nand_chip *chip = mtd_to_nand(mtd); struct hinfc_host *host = nand_get_controller_data(chip); if (chipselect < 0) @@ -364,9 +363,8 @@ static void hisi_nfc_select_chip(struct mtd_info *mtd, int chipselect) host->chipselect = chipselect; } -static uint8_t hisi_nfc_read_byte(struct mtd_info *mtd) +static uint8_t hisi_nfc_read_byte(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct hinfc_host *host = nand_get_controller_data(chip); if (host->command == NAND_CMD_STATUS) @@ -380,28 +378,17 @@ static uint8_t hisi_nfc_read_byte(struct mtd_info *mtd) return *(uint8_t *)(host->buffer + host->offset - 1); } -static u16 hisi_nfc_read_word(struct mtd_info *mtd) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - struct hinfc_host *host = nand_get_controller_data(chip); - - host->offset += 2; - return *(u16 *)(host->buffer + host->offset - 2); -} - static void -hisi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len) +hisi_nfc_write_buf(struct nand_chip *chip, const uint8_t *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct hinfc_host *host = nand_get_controller_data(chip); memcpy(host->buffer + host->offset, buf, len); host->offset += len; } -static void hisi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) +static void hisi_nfc_read_buf(struct nand_chip *chip, uint8_t *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct hinfc_host *host = nand_get_controller_data(chip); memcpy(buf, host->buffer + host->offset, len); @@ -442,10 +429,10 @@ static void set_addr(struct mtd_info *mtd, int column, int page_addr) } } -static void hisi_nfc_cmdfunc(struct mtd_info *mtd, unsigned command, int column, - int page_addr) +static void hisi_nfc_cmdfunc(struct nand_chip *chip, unsigned command, + int column, int page_addr) { - struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(chip); struct hinfc_host *host = nand_get_controller_data(chip); int is_cache_invalid = 1; unsigned int flag = 0; @@ -537,15 +524,16 @@ static irqreturn_t hinfc_irq_handle(int irq, void *devid) return IRQ_HANDLED; } -static int hisi_nand_read_page_hwecc(struct mtd_info *mtd, - struct nand_chip *chip, uint8_t *buf, int oob_required, int page) +static int hisi_nand_read_page_hwecc(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct hinfc_host *host = nand_get_controller_data(chip); int max_bitflips = 0, stat = 0, stat_max = 0, status_ecc; int stat_1, stat_2; nand_read_page_op(chip, page, 0, buf, mtd->writesize); - chip->read_buf(mtd, chip->oob_poi, mtd->oobsize); + chip->legacy.read_buf(chip, chip->oob_poi, mtd->oobsize); /* errors which can not be corrected by ECC */ if (host->irq_status & HINFC504_INTS_UE) { @@ -569,9 +557,9 @@ static int hisi_nand_read_page_hwecc(struct mtd_info *mtd, return max_bitflips; } -static int hisi_nand_read_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int hisi_nand_read_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct hinfc_host *host = nand_get_controller_data(chip); nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize); @@ -585,13 +573,15 @@ static int hisi_nand_read_oob(struct mtd_info *mtd, struct nand_chip *chip, return 0; } -static int hisi_nand_write_page_hwecc(struct mtd_info *mtd, - struct nand_chip *chip, const uint8_t *buf, int oob_required, - int page) +static int hisi_nand_write_page_hwecc(struct nand_chip *chip, + const uint8_t *buf, int oob_required, + int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize); if (oob_required) - chip->write_buf(mtd, chip->oob_poi, mtd->oobsize); + chip->legacy.write_buf(chip, chip->oob_poi, mtd->oobsize); return nand_prog_page_end_op(chip); } @@ -792,15 +782,14 @@ static int hisi_nfc_probe(struct platform_device *pdev) nand_set_controller_data(chip, host); nand_set_flash_node(chip, np); - chip->cmdfunc = hisi_nfc_cmdfunc; + chip->legacy.cmdfunc = hisi_nfc_cmdfunc; chip->select_chip = hisi_nfc_select_chip; - chip->read_byte = hisi_nfc_read_byte; - chip->read_word = hisi_nfc_read_word; - chip->write_buf = hisi_nfc_write_buf; - chip->read_buf = hisi_nfc_read_buf; - chip->chip_delay = HINFC504_CHIP_DELAY; - chip->set_features = nand_get_set_features_notsupp; - chip->get_features = nand_get_set_features_notsupp; + chip->legacy.read_byte = hisi_nfc_read_byte; + chip->legacy.write_buf = hisi_nfc_write_buf; + chip->legacy.read_buf = hisi_nfc_read_buf; + chip->legacy.chip_delay = HINFC504_CHIP_DELAY; + chip->legacy.set_features = nand_get_set_features_notsupp; + chip->legacy.get_features = nand_get_set_features_notsupp; hisi_nfc_host_init(host); @@ -811,7 +800,7 @@ static int hisi_nfc_probe(struct platform_device *pdev) } chip->dummy_controller.ops = &hisi_nfc_controller_ops; - ret = nand_scan(mtd, max_chips); + ret = nand_scan(chip, max_chips); if (ret) return ret; @@ -828,9 +817,8 @@ static int hisi_nfc_probe(struct platform_device *pdev) static int hisi_nfc_remove(struct platform_device *pdev) { struct hinfc_host *host = platform_get_drvdata(pdev); - struct mtd_info *mtd = nand_to_mtd(&host->chip); - nand_release(mtd); + nand_release(&host->chip); return 0; } diff --git a/drivers/mtd/nand/raw/internals.h b/drivers/mtd/nand/raw/internals.h new file mode 100644 index 000000000000..04c2cf74eff3 --- /dev/null +++ b/drivers/mtd/nand/raw/internals.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2018 - Bootlin + * + * Author: Boris Brezillon <boris.brezillon@bootlin.com> + * + * Header containing internal definitions to be used only by core files. + * NAND controller drivers should not include this file. + */ + +#ifndef __LINUX_RAWNAND_INTERNALS +#define __LINUX_RAWNAND_INTERNALS + +#include <linux/mtd/rawnand.h> + +/* + * NAND Flash Manufacturer ID Codes + */ +#define NAND_MFR_AMD 0x01 +#define NAND_MFR_ATO 0x9b +#define NAND_MFR_EON 0x92 +#define NAND_MFR_ESMT 0xc8 +#define NAND_MFR_FUJITSU 0x04 +#define NAND_MFR_HYNIX 0xad +#define NAND_MFR_INTEL 0x89 +#define NAND_MFR_MACRONIX 0xc2 +#define NAND_MFR_MICRON 0x2c +#define NAND_MFR_NATIONAL 0x8f +#define NAND_MFR_RENESAS 0x07 +#define NAND_MFR_SAMSUNG 0xec +#define NAND_MFR_SANDISK 0x45 +#define NAND_MFR_STMICRO 0x20 +#define NAND_MFR_TOSHIBA 0x98 +#define NAND_MFR_WINBOND 0xef + +/** + * struct nand_manufacturer_ops - NAND Manufacturer operations + * @detect: detect the NAND memory organization and capabilities + * @init: initialize all vendor specific fields (like the ->read_retry() + * implementation) if any. + * @cleanup: the ->init() function may have allocated resources, ->cleanup() + * is here to let vendor specific code release those resources. + * @fixup_onfi_param_page: apply vendor specific fixups to the ONFI parameter + * page. This is called after the checksum is verified. + */ +struct nand_manufacturer_ops { + void (*detect)(struct nand_chip *chip); + int (*init)(struct nand_chip *chip); + void (*cleanup)(struct nand_chip *chip); + void (*fixup_onfi_param_page)(struct nand_chip *chip, + struct nand_onfi_params *p); +}; + +/** + * struct nand_manufacturer - NAND Flash Manufacturer structure + * @name: Manufacturer name + * @id: manufacturer ID code of device. + * @ops: manufacturer operations + */ +struct nand_manufacturer { + int id; + char *name; + const struct nand_manufacturer_ops *ops; +}; + + +extern struct nand_flash_dev nand_flash_ids[]; + +extern const struct nand_manufacturer_ops amd_nand_manuf_ops; +extern const struct nand_manufacturer_ops esmt_nand_manuf_ops; +extern const struct nand_manufacturer_ops hynix_nand_manuf_ops; +extern const struct nand_manufacturer_ops macronix_nand_manuf_ops; +extern const struct nand_manufacturer_ops micron_nand_manuf_ops; +extern const struct nand_manufacturer_ops samsung_nand_manuf_ops; +extern const struct nand_manufacturer_ops toshiba_nand_manuf_ops; + +/* Core functions */ +const struct nand_manufacturer *nand_get_manufacturer(u8 id); +int nand_markbad_bbm(struct nand_chip *chip, loff_t ofs); +int nand_erase_nand(struct nand_chip *chip, struct erase_info *instr, + int allowbbt); +int onfi_fill_data_interface(struct nand_chip *chip, + enum nand_data_interface_type type, + int timing_mode); +int nand_get_features(struct nand_chip *chip, int addr, u8 *subfeature_param); +int nand_set_features(struct nand_chip *chip, int addr, u8 *subfeature_param); +int nand_read_page_raw_notsupp(struct nand_chip *chip, u8 *buf, + int oob_required, int page); +int nand_write_page_raw_notsupp(struct nand_chip *chip, const u8 *buf, + int oob_required, int page); +int nand_exit_status_op(struct nand_chip *chip); +int nand_read_param_page_op(struct nand_chip *chip, u8 page, void *buf, + unsigned int len); +void nand_decode_ext_id(struct nand_chip *chip); +void panic_nand_wait(struct nand_chip *chip, unsigned long timeo); +void sanitize_string(uint8_t *s, size_t len); + +/* BBT functions */ +int nand_markbad_bbt(struct nand_chip *chip, loff_t offs); +int nand_isreserved_bbt(struct nand_chip *chip, loff_t offs); +int nand_isbad_bbt(struct nand_chip *chip, loff_t offs, int allowbbt); + +/* Legacy */ +void nand_legacy_set_defaults(struct nand_chip *chip); +void nand_legacy_adjust_cmdfunc(struct nand_chip *chip); +int nand_legacy_check_hooks(struct nand_chip *chip); + +/* ONFI functions */ +u16 onfi_crc16(u16 crc, u8 const *p, size_t len); +int nand_onfi_detect(struct nand_chip *chip); + +/* JEDEC functions */ +int nand_jedec_detect(struct nand_chip *chip); + +#endif /* __LINUX_RAWNAND_INTERNALS */ diff --git a/drivers/mtd/nand/raw/jz4740_nand.c b/drivers/mtd/nand/raw/jz4740_nand.c index a7515452bc59..fb59cfca11a7 100644 --- a/drivers/mtd/nand/raw/jz4740_nand.c +++ b/drivers/mtd/nand/raw/jz4740_nand.c @@ -78,10 +78,9 @@ static inline struct jz_nand *mtd_to_jz_nand(struct mtd_info *mtd) return container_of(mtd_to_nand(mtd), struct jz_nand, chip); } -static void jz_nand_select_chip(struct mtd_info *mtd, int chipnr) +static void jz_nand_select_chip(struct nand_chip *chip, int chipnr) { - struct jz_nand *nand = mtd_to_jz_nand(mtd); - struct nand_chip *chip = mtd_to_nand(mtd); + struct jz_nand *nand = mtd_to_jz_nand(nand_to_mtd(chip)); uint32_t ctrl; int banknr; @@ -92,18 +91,18 @@ static void jz_nand_select_chip(struct mtd_info *mtd, int chipnr) banknr = -1; } else { banknr = nand->banks[chipnr] - 1; - chip->IO_ADDR_R = nand->bank_base[banknr]; - chip->IO_ADDR_W = nand->bank_base[banknr]; + chip->legacy.IO_ADDR_R = nand->bank_base[banknr]; + chip->legacy.IO_ADDR_W = nand->bank_base[banknr]; } writel(ctrl, nand->base + JZ_REG_NAND_CTRL); nand->selected_bank = banknr; } -static void jz_nand_cmd_ctrl(struct mtd_info *mtd, int dat, unsigned int ctrl) +static void jz_nand_cmd_ctrl(struct nand_chip *chip, int dat, + unsigned int ctrl) { - struct jz_nand *nand = mtd_to_jz_nand(mtd); - struct nand_chip *chip = mtd_to_nand(mtd); + struct jz_nand *nand = mtd_to_jz_nand(nand_to_mtd(chip)); uint32_t reg; void __iomem *bank_base = nand->bank_base[nand->selected_bank]; @@ -115,7 +114,7 @@ static void jz_nand_cmd_ctrl(struct mtd_info *mtd, int dat, unsigned int ctrl) bank_base += JZ_NAND_MEM_ADDR_OFFSET; else if (ctrl & NAND_CLE) bank_base += JZ_NAND_MEM_CMD_OFFSET; - chip->IO_ADDR_W = bank_base; + chip->legacy.IO_ADDR_W = bank_base; reg = readl(nand->base + JZ_REG_NAND_CTRL); if (ctrl & NAND_NCE) @@ -125,18 +124,18 @@ static void jz_nand_cmd_ctrl(struct mtd_info *mtd, int dat, unsigned int ctrl) writel(reg, nand->base + JZ_REG_NAND_CTRL); } if (dat != NAND_CMD_NONE) - writeb(dat, chip->IO_ADDR_W); + writeb(dat, chip->legacy.IO_ADDR_W); } -static int jz_nand_dev_ready(struct mtd_info *mtd) +static int jz_nand_dev_ready(struct nand_chip *chip) { - struct jz_nand *nand = mtd_to_jz_nand(mtd); + struct jz_nand *nand = mtd_to_jz_nand(nand_to_mtd(chip)); return gpiod_get_value_cansleep(nand->busy_gpio); } -static void jz_nand_hwctl(struct mtd_info *mtd, int mode) +static void jz_nand_hwctl(struct nand_chip *chip, int mode) { - struct jz_nand *nand = mtd_to_jz_nand(mtd); + struct jz_nand *nand = mtd_to_jz_nand(nand_to_mtd(chip)); uint32_t reg; writel(0, nand->base + JZ_REG_NAND_IRQ_STAT); @@ -162,10 +161,10 @@ static void jz_nand_hwctl(struct mtd_info *mtd, int mode) writel(reg, nand->base + JZ_REG_NAND_ECC_CTRL); } -static int jz_nand_calculate_ecc_rs(struct mtd_info *mtd, const uint8_t *dat, - uint8_t *ecc_code) +static int jz_nand_calculate_ecc_rs(struct nand_chip *chip, const uint8_t *dat, + uint8_t *ecc_code) { - struct jz_nand *nand = mtd_to_jz_nand(mtd); + struct jz_nand *nand = mtd_to_jz_nand(nand_to_mtd(chip)); uint32_t reg, status; int i; unsigned int timeout = 1000; @@ -215,10 +214,10 @@ static void jz_nand_correct_data(uint8_t *dat, int index, int mask) dat[index+1] = (data >> 8) & 0xff; } -static int jz_nand_correct_ecc_rs(struct mtd_info *mtd, uint8_t *dat, - uint8_t *read_ecc, uint8_t *calc_ecc) +static int jz_nand_correct_ecc_rs(struct nand_chip *chip, uint8_t *dat, + uint8_t *read_ecc, uint8_t *calc_ecc) { - struct jz_nand *nand = mtd_to_jz_nand(mtd); + struct jz_nand *nand = mtd_to_jz_nand(nand_to_mtd(chip)); int i, error_count, index; uint32_t reg, status, error; unsigned int timeout = 1000; @@ -331,19 +330,19 @@ static int jz_nand_detect_bank(struct platform_device *pdev, if (chipnr == 0) { /* Detect first chip. */ - ret = nand_scan(mtd, 1); + ret = nand_scan(chip, 1); if (ret) goto notfound_id; /* Retrieve the IDs from the first chip. */ - chip->select_chip(mtd, 0); + chip->select_chip(chip, 0); nand_reset_op(chip); nand_readid_op(chip, 0, id, sizeof(id)); *nand_maf_id = id[0]; *nand_dev_id = id[1]; } else { /* Detect additional chip. */ - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, chipnr); nand_reset_op(chip); nand_readid_op(chip, 0, id, sizeof(id)); if (*nand_maf_id != id[0] || *nand_dev_id != id[1]) { @@ -426,13 +425,13 @@ static int jz_nand_probe(struct platform_device *pdev) chip->ecc.strength = 4; chip->ecc.options = NAND_ECC_GENERIC_ERASED_CHECK; - chip->chip_delay = 50; - chip->cmd_ctrl = jz_nand_cmd_ctrl; + chip->legacy.chip_delay = 50; + chip->legacy.cmd_ctrl = jz_nand_cmd_ctrl; chip->select_chip = jz_nand_select_chip; chip->dummy_controller.ops = &jz_nand_controller_ops; if (nand->busy_gpio) - chip->dev_ready = jz_nand_dev_ready; + chip->legacy.dev_ready = jz_nand_dev_ready; platform_set_drvdata(pdev, nand); @@ -507,7 +506,7 @@ static int jz_nand_remove(struct platform_device *pdev) struct jz_nand *nand = platform_get_drvdata(pdev); size_t i; - nand_release(nand_to_mtd(&nand->chip)); + nand_release(&nand->chip); /* Deassert and disable all chips */ writel(0, nand->base + JZ_REG_NAND_CTRL); diff --git a/drivers/mtd/nand/raw/jz4780_nand.c b/drivers/mtd/nand/raw/jz4780_nand.c index db4fa60bd52a..cdf22100ab77 100644 --- a/drivers/mtd/nand/raw/jz4780_nand.c +++ b/drivers/mtd/nand/raw/jz4780_nand.c @@ -71,9 +71,9 @@ static inline struct jz4780_nand_controller return container_of(ctrl, struct jz4780_nand_controller, controller); } -static void jz4780_nand_select_chip(struct mtd_info *mtd, int chipnr) +static void jz4780_nand_select_chip(struct nand_chip *chip, int chipnr) { - struct jz4780_nand_chip *nand = to_jz4780_nand_chip(mtd); + struct jz4780_nand_chip *nand = to_jz4780_nand_chip(nand_to_mtd(chip)); struct jz4780_nand_controller *nfc = to_jz4780_nand_controller(nand->chip.controller); struct jz4780_nand_cs *cs; @@ -86,10 +86,10 @@ static void jz4780_nand_select_chip(struct mtd_info *mtd, int chipnr) nfc->selected = chipnr; } -static void jz4780_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, +static void jz4780_nand_cmd_ctrl(struct nand_chip *chip, int cmd, unsigned int ctrl) { - struct jz4780_nand_chip *nand = to_jz4780_nand_chip(mtd); + struct jz4780_nand_chip *nand = to_jz4780_nand_chip(nand_to_mtd(chip)); struct jz4780_nand_controller *nfc = to_jz4780_nand_controller(nand->chip.controller); struct jz4780_nand_cs *cs; @@ -109,24 +109,24 @@ static void jz4780_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, writeb(cmd, cs->base + OFFSET_CMD); } -static int jz4780_nand_dev_ready(struct mtd_info *mtd) +static int jz4780_nand_dev_ready(struct nand_chip *chip) { - struct jz4780_nand_chip *nand = to_jz4780_nand_chip(mtd); + struct jz4780_nand_chip *nand = to_jz4780_nand_chip(nand_to_mtd(chip)); return !gpiod_get_value_cansleep(nand->busy_gpio); } -static void jz4780_nand_ecc_hwctl(struct mtd_info *mtd, int mode) +static void jz4780_nand_ecc_hwctl(struct nand_chip *chip, int mode) { - struct jz4780_nand_chip *nand = to_jz4780_nand_chip(mtd); + struct jz4780_nand_chip *nand = to_jz4780_nand_chip(nand_to_mtd(chip)); nand->reading = (mode == NAND_ECC_READ); } -static int jz4780_nand_ecc_calculate(struct mtd_info *mtd, const u8 *dat, +static int jz4780_nand_ecc_calculate(struct nand_chip *chip, const u8 *dat, u8 *ecc_code) { - struct jz4780_nand_chip *nand = to_jz4780_nand_chip(mtd); + struct jz4780_nand_chip *nand = to_jz4780_nand_chip(nand_to_mtd(chip)); struct jz4780_nand_controller *nfc = to_jz4780_nand_controller(nand->chip.controller); struct jz4780_bch_params params; @@ -144,10 +144,10 @@ static int jz4780_nand_ecc_calculate(struct mtd_info *mtd, const u8 *dat, return jz4780_bch_calculate(nfc->bch, ¶ms, dat, ecc_code); } -static int jz4780_nand_ecc_correct(struct mtd_info *mtd, u8 *dat, +static int jz4780_nand_ecc_correct(struct nand_chip *chip, u8 *dat, u8 *read_ecc, u8 *calc_ecc) { - struct jz4780_nand_chip *nand = to_jz4780_nand_chip(mtd); + struct jz4780_nand_chip *nand = to_jz4780_nand_chip(nand_to_mtd(chip)); struct jz4780_nand_controller *nfc = to_jz4780_nand_controller(nand->chip.controller); struct jz4780_bch_params params; @@ -256,7 +256,7 @@ static int jz4780_nand_init_chip(struct platform_device *pdev, dev_err(dev, "failed to request busy GPIO: %d\n", ret); return ret; } else if (nand->busy_gpio) { - nand->chip.dev_ready = jz4780_nand_dev_ready; + nand->chip.legacy.dev_ready = jz4780_nand_dev_ready; } nand->wp_gpio = devm_gpiod_get_optional(dev, "wp", GPIOD_OUT_LOW); @@ -275,24 +275,24 @@ static int jz4780_nand_init_chip(struct platform_device *pdev, return -ENOMEM; mtd->dev.parent = dev; - chip->IO_ADDR_R = cs->base + OFFSET_DATA; - chip->IO_ADDR_W = cs->base + OFFSET_DATA; - chip->chip_delay = RB_DELAY_US; + chip->legacy.IO_ADDR_R = cs->base + OFFSET_DATA; + chip->legacy.IO_ADDR_W = cs->base + OFFSET_DATA; + chip->legacy.chip_delay = RB_DELAY_US; chip->options = NAND_NO_SUBPAGE_WRITE; chip->select_chip = jz4780_nand_select_chip; - chip->cmd_ctrl = jz4780_nand_cmd_ctrl; + chip->legacy.cmd_ctrl = jz4780_nand_cmd_ctrl; chip->ecc.mode = NAND_ECC_HW; chip->controller = &nfc->controller; nand_set_flash_node(chip, np); chip->controller->ops = &jz4780_nand_controller_ops; - ret = nand_scan(mtd, 1); + ret = nand_scan(chip, 1); if (ret) return ret; ret = mtd_device_register(mtd, NULL, 0); if (ret) { - nand_release(mtd); + nand_release(chip); return ret; } @@ -307,7 +307,7 @@ static void jz4780_nand_cleanup_chips(struct jz4780_nand_controller *nfc) while (!list_empty(&nfc->chips)) { chip = list_first_entry(&nfc->chips, struct jz4780_nand_chip, chip_list); - nand_release(nand_to_mtd(&chip->chip)); + nand_release(&chip->chip); list_del(&chip->chip_list); } } @@ -352,7 +352,7 @@ static int jz4780_nand_probe(struct platform_device *pdev) return -ENODEV; } - nfc = devm_kzalloc(dev, sizeof(*nfc) + (sizeof(nfc->cs[0]) * num_banks), GFP_KERNEL); + nfc = devm_kzalloc(dev, struct_size(nfc, cs, num_banks), GFP_KERNEL); if (!nfc) return -ENOMEM; diff --git a/drivers/mtd/nand/raw/lpc32xx_mlc.c b/drivers/mtd/nand/raw/lpc32xx_mlc.c index e82abada130a..abbb655fe154 100644 --- a/drivers/mtd/nand/raw/lpc32xx_mlc.c +++ b/drivers/mtd/nand/raw/lpc32xx_mlc.c @@ -286,10 +286,9 @@ static void lpc32xx_nand_setup(struct lpc32xx_nand_host *host) /* * Hardware specific access to control lines */ -static void lpc32xx_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, +static void lpc32xx_nand_cmd_ctrl(struct nand_chip *nand_chip, int cmd, unsigned int ctrl) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); struct lpc32xx_nand_host *host = nand_get_controller_data(nand_chip); if (cmd != NAND_CMD_NONE) { @@ -303,9 +302,8 @@ static void lpc32xx_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, /* * Read Device Ready (NAND device _and_ controller ready) */ -static int lpc32xx_nand_device_ready(struct mtd_info *mtd) +static int lpc32xx_nand_device_ready(struct nand_chip *nand_chip) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); struct lpc32xx_nand_host *host = nand_get_controller_data(nand_chip); if ((readb(MLC_ISR(host->io_base)) & @@ -330,8 +328,9 @@ static irqreturn_t lpc3xxx_nand_irq(int irq, struct lpc32xx_nand_host *host) return IRQ_HANDLED; } -static int lpc32xx_waitfunc_nand(struct mtd_info *mtd, struct nand_chip *chip) +static int lpc32xx_waitfunc_nand(struct nand_chip *chip) { + struct mtd_info *mtd = nand_to_mtd(chip); struct lpc32xx_nand_host *host = nand_get_controller_data(chip); if (readb(MLC_ISR(host->io_base)) & MLCISR_NAND_READY) @@ -349,9 +348,9 @@ exit: return NAND_STATUS_READY; } -static int lpc32xx_waitfunc_controller(struct mtd_info *mtd, - struct nand_chip *chip) +static int lpc32xx_waitfunc_controller(struct nand_chip *chip) { + struct mtd_info *mtd = nand_to_mtd(chip); struct lpc32xx_nand_host *host = nand_get_controller_data(chip); if (readb(MLC_ISR(host->io_base)) & MLCISR_CONTROLLER_READY) @@ -369,10 +368,10 @@ exit: return NAND_STATUS_READY; } -static int lpc32xx_waitfunc(struct mtd_info *mtd, struct nand_chip *chip) +static int lpc32xx_waitfunc(struct nand_chip *chip) { - lpc32xx_waitfunc_nand(mtd, chip); - lpc32xx_waitfunc_controller(mtd, chip); + lpc32xx_waitfunc_nand(chip); + lpc32xx_waitfunc_controller(chip); return NAND_STATUS_READY; } @@ -442,9 +441,10 @@ out1: return -ENXIO; } -static int lpc32xx_read_page(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int lpc32xx_read_page(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct lpc32xx_nand_host *host = nand_get_controller_data(chip); int i, j; uint8_t *oobbuf = chip->oob_poi; @@ -470,7 +470,7 @@ static int lpc32xx_read_page(struct mtd_info *mtd, struct nand_chip *chip, writeb(0x00, MLC_ECC_AUTO_DEC_REG(host->io_base)); /* Wait for Controller Ready */ - lpc32xx_waitfunc_controller(mtd, chip); + lpc32xx_waitfunc_controller(chip); /* Check ECC Error status */ mlc_isr = readl(MLC_ISR(host->io_base)); @@ -507,11 +507,11 @@ static int lpc32xx_read_page(struct mtd_info *mtd, struct nand_chip *chip, return 0; } -static int lpc32xx_write_page_lowlevel(struct mtd_info *mtd, - struct nand_chip *chip, +static int lpc32xx_write_page_lowlevel(struct nand_chip *chip, const uint8_t *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct lpc32xx_nand_host *host = nand_get_controller_data(chip); const uint8_t *oobbuf = chip->oob_poi; uint8_t *dma_buf = (uint8_t *)buf; @@ -551,32 +551,30 @@ static int lpc32xx_write_page_lowlevel(struct mtd_info *mtd, writeb(0x00, MLC_ECC_AUTO_ENC_REG(host->io_base)); /* Wait for Controller Ready */ - lpc32xx_waitfunc_controller(mtd, chip); + lpc32xx_waitfunc_controller(chip); } return nand_prog_page_end_op(chip); } -static int lpc32xx_read_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int lpc32xx_read_oob(struct nand_chip *chip, int page) { struct lpc32xx_nand_host *host = nand_get_controller_data(chip); /* Read whole page - necessary with MLC controller! */ - lpc32xx_read_page(mtd, chip, host->dummy_buf, 1, page); + lpc32xx_read_page(chip, host->dummy_buf, 1, page); return 0; } -static int lpc32xx_write_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int lpc32xx_write_oob(struct nand_chip *chip, int page) { /* None, write_oob conflicts with the automatic LPC MLC ECC decoder! */ return 0; } /* Prepares MLC for transfers with H/W ECC enabled: always enabled anyway */ -static void lpc32xx_ecc_enable(struct mtd_info *mtd, int mode) +static void lpc32xx_ecc_enable(struct nand_chip *chip, int mode) { /* Always enabled! */ } @@ -741,11 +739,11 @@ static int lpc32xx_nand_probe(struct platform_device *pdev) if (res) goto put_clk; - nand_chip->cmd_ctrl = lpc32xx_nand_cmd_ctrl; - nand_chip->dev_ready = lpc32xx_nand_device_ready; - nand_chip->chip_delay = 25; /* us */ - nand_chip->IO_ADDR_R = MLC_DATA(host->io_base); - nand_chip->IO_ADDR_W = MLC_DATA(host->io_base); + nand_chip->legacy.cmd_ctrl = lpc32xx_nand_cmd_ctrl; + nand_chip->legacy.dev_ready = lpc32xx_nand_device_ready; + nand_chip->legacy.chip_delay = 25; /* us */ + nand_chip->legacy.IO_ADDR_R = MLC_DATA(host->io_base); + nand_chip->legacy.IO_ADDR_W = MLC_DATA(host->io_base); /* Init NAND controller */ lpc32xx_nand_setup(host); @@ -762,7 +760,7 @@ static int lpc32xx_nand_probe(struct platform_device *pdev) nand_chip->ecc.read_oob = lpc32xx_read_oob; nand_chip->ecc.strength = 4; nand_chip->ecc.bytes = 10; - nand_chip->waitfunc = lpc32xx_waitfunc; + nand_chip->legacy.waitfunc = lpc32xx_waitfunc; nand_chip->options = NAND_NO_SUBPAGE_WRITE; nand_chip->bbt_options = NAND_BBT_USE_FLASH | NAND_BBT_NO_OOB; @@ -802,7 +800,7 @@ static int lpc32xx_nand_probe(struct platform_device *pdev) * SMALL block or LARGE block. */ nand_chip->dummy_controller.ops = &lpc32xx_nand_controller_ops; - res = nand_scan(mtd, 1); + res = nand_scan(nand_chip, 1); if (res) goto free_irq; @@ -839,9 +837,8 @@ free_gpio: static int lpc32xx_nand_remove(struct platform_device *pdev) { struct lpc32xx_nand_host *host = platform_get_drvdata(pdev); - struct mtd_info *mtd = nand_to_mtd(&host->nand_chip); - nand_release(mtd); + nand_release(&host->nand_chip); free_irq(host->irq, host); if (use_dma) dma_release_channel(host->dma_chan); diff --git a/drivers/mtd/nand/raw/lpc32xx_slc.c b/drivers/mtd/nand/raw/lpc32xx_slc.c index a4e8b7e75135..f2f2cdbb9d04 100644 --- a/drivers/mtd/nand/raw/lpc32xx_slc.c +++ b/drivers/mtd/nand/raw/lpc32xx_slc.c @@ -278,11 +278,10 @@ static void lpc32xx_nand_setup(struct lpc32xx_nand_host *host) /* * Hardware specific access to control lines */ -static void lpc32xx_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, - unsigned int ctrl) +static void lpc32xx_nand_cmd_ctrl(struct nand_chip *chip, int cmd, + unsigned int ctrl) { uint32_t tmp; - struct nand_chip *chip = mtd_to_nand(mtd); struct lpc32xx_nand_host *host = nand_get_controller_data(chip); /* Does CE state need to be changed? */ @@ -304,9 +303,8 @@ static void lpc32xx_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, /* * Read the Device Ready pin */ -static int lpc32xx_nand_device_ready(struct mtd_info *mtd) +static int lpc32xx_nand_device_ready(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct lpc32xx_nand_host *host = nand_get_controller_data(chip); int rdy = 0; @@ -337,7 +335,7 @@ static void lpc32xx_wp_disable(struct lpc32xx_nand_host *host) /* * Prepares SLC for transfers with H/W ECC enabled */ -static void lpc32xx_nand_ecc_enable(struct mtd_info *mtd, int mode) +static void lpc32xx_nand_ecc_enable(struct nand_chip *chip, int mode) { /* Hardware ECC is enabled automatically in hardware as needed */ } @@ -345,7 +343,7 @@ static void lpc32xx_nand_ecc_enable(struct mtd_info *mtd, int mode) /* * Calculates the ECC for the data */ -static int lpc32xx_nand_ecc_calculate(struct mtd_info *mtd, +static int lpc32xx_nand_ecc_calculate(struct nand_chip *chip, const unsigned char *buf, unsigned char *code) { @@ -359,9 +357,8 @@ static int lpc32xx_nand_ecc_calculate(struct mtd_info *mtd, /* * Read a single byte from NAND device */ -static uint8_t lpc32xx_nand_read_byte(struct mtd_info *mtd) +static uint8_t lpc32xx_nand_read_byte(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct lpc32xx_nand_host *host = nand_get_controller_data(chip); return (uint8_t)readl(SLC_DATA(host->io_base)); @@ -370,9 +367,8 @@ static uint8_t lpc32xx_nand_read_byte(struct mtd_info *mtd) /* * Simple device read without ECC */ -static void lpc32xx_nand_read_buf(struct mtd_info *mtd, u_char *buf, int len) +static void lpc32xx_nand_read_buf(struct nand_chip *chip, u_char *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct lpc32xx_nand_host *host = nand_get_controller_data(chip); /* Direct device read with no ECC */ @@ -383,9 +379,9 @@ static void lpc32xx_nand_read_buf(struct mtd_info *mtd, u_char *buf, int len) /* * Simple device write without ECC */ -static void lpc32xx_nand_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len) +static void lpc32xx_nand_write_buf(struct nand_chip *chip, const uint8_t *buf, + int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct lpc32xx_nand_host *host = nand_get_controller_data(chip); /* Direct device write with no ECC */ @@ -396,18 +392,20 @@ static void lpc32xx_nand_write_buf(struct mtd_info *mtd, const uint8_t *buf, int /* * Read the OOB data from the device without ECC using FIFO method */ -static int lpc32xx_nand_read_oob_syndrome(struct mtd_info *mtd, - struct nand_chip *chip, int page) +static int lpc32xx_nand_read_oob_syndrome(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + return nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize); } /* * Write the OOB data to the device without ECC using FIFO method */ -static int lpc32xx_nand_write_oob_syndrome(struct mtd_info *mtd, - struct nand_chip *chip, int page) +static int lpc32xx_nand_write_oob_syndrome(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + return nand_prog_page_op(chip, page, mtd->writesize, chip->oob_poi, mtd->oobsize); } @@ -610,10 +608,10 @@ static int lpc32xx_xfer(struct mtd_info *mtd, uint8_t *buf, int eccsubpages, * Read the data and OOB data from the device, use ECC correction with the * data, disable ECC for the OOB data */ -static int lpc32xx_nand_read_page_syndrome(struct mtd_info *mtd, - struct nand_chip *chip, uint8_t *buf, +static int lpc32xx_nand_read_page_syndrome(struct nand_chip *chip, uint8_t *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct lpc32xx_nand_host *host = nand_get_controller_data(chip); struct mtd_oob_region oobregion = { }; int stat, i, status, error; @@ -626,7 +624,7 @@ static int lpc32xx_nand_read_page_syndrome(struct mtd_info *mtd, status = lpc32xx_xfer(mtd, buf, chip->ecc.steps, 1); /* Get OOB data */ - chip->read_buf(mtd, chip->oob_poi, mtd->oobsize); + chip->legacy.read_buf(chip, chip->oob_poi, mtd->oobsize); /* Convert to stored ECC format */ lpc32xx_slc_ecc_copy(tmpecc, (uint32_t *) host->ecc_buf, chip->ecc.steps); @@ -639,7 +637,7 @@ static int lpc32xx_nand_read_page_syndrome(struct mtd_info *mtd, oobecc = chip->oob_poi + oobregion.offset; for (i = 0; i < chip->ecc.steps; i++) { - stat = chip->ecc.correct(mtd, buf, oobecc, + stat = chip->ecc.correct(chip, buf, oobecc, &tmpecc[i * chip->ecc.bytes]); if (stat < 0) mtd->ecc_stats.failed++; @@ -657,17 +655,18 @@ static int lpc32xx_nand_read_page_syndrome(struct mtd_info *mtd, * Read the data and OOB data from the device, no ECC correction with the * data or OOB data */ -static int lpc32xx_nand_read_page_raw_syndrome(struct mtd_info *mtd, - struct nand_chip *chip, +static int lpc32xx_nand_read_page_raw_syndrome(struct nand_chip *chip, uint8_t *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + /* Issue read command */ nand_read_page_op(chip, page, 0, NULL, 0); /* Raw reads can just use the FIFO interface */ - chip->read_buf(mtd, buf, chip->ecc.size * chip->ecc.steps); - chip->read_buf(mtd, chip->oob_poi, mtd->oobsize); + chip->legacy.read_buf(chip, buf, chip->ecc.size * chip->ecc.steps); + chip->legacy.read_buf(chip, chip->oob_poi, mtd->oobsize); return 0; } @@ -676,11 +675,11 @@ static int lpc32xx_nand_read_page_raw_syndrome(struct mtd_info *mtd, * Write the data and OOB data to the device, use ECC with the data, * disable ECC for the OOB data */ -static int lpc32xx_nand_write_page_syndrome(struct mtd_info *mtd, - struct nand_chip *chip, +static int lpc32xx_nand_write_page_syndrome(struct nand_chip *chip, const uint8_t *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct lpc32xx_nand_host *host = nand_get_controller_data(chip); struct mtd_oob_region oobregion = { }; uint8_t *pb; @@ -705,7 +704,7 @@ static int lpc32xx_nand_write_page_syndrome(struct mtd_info *mtd, lpc32xx_slc_ecc_copy(pb, (uint32_t *)host->ecc_buf, chip->ecc.steps); /* Write ECC data to device */ - chip->write_buf(mtd, chip->oob_poi, mtd->oobsize); + chip->legacy.write_buf(chip, chip->oob_poi, mtd->oobsize); return nand_prog_page_end_op(chip); } @@ -714,15 +713,16 @@ static int lpc32xx_nand_write_page_syndrome(struct mtd_info *mtd, * Write the data and OOB data to the device, no ECC correction with the * data or OOB data */ -static int lpc32xx_nand_write_page_raw_syndrome(struct mtd_info *mtd, - struct nand_chip *chip, +static int lpc32xx_nand_write_page_raw_syndrome(struct nand_chip *chip, const uint8_t *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + /* Raw writes can just use the FIFO interface */ nand_prog_page_begin_op(chip, page, 0, buf, chip->ecc.size * chip->ecc.steps); - chip->write_buf(mtd, chip->oob_poi, mtd->oobsize); + chip->legacy.write_buf(chip, chip->oob_poi, mtd->oobsize); return nand_prog_page_end_op(chip); } @@ -878,11 +878,11 @@ static int lpc32xx_nand_probe(struct platform_device *pdev) goto enable_wp; /* Set NAND IO addresses and command/ready functions */ - chip->IO_ADDR_R = SLC_DATA(host->io_base); - chip->IO_ADDR_W = SLC_DATA(host->io_base); - chip->cmd_ctrl = lpc32xx_nand_cmd_ctrl; - chip->dev_ready = lpc32xx_nand_device_ready; - chip->chip_delay = 20; /* 20us command delay time */ + chip->legacy.IO_ADDR_R = SLC_DATA(host->io_base); + chip->legacy.IO_ADDR_W = SLC_DATA(host->io_base); + chip->legacy.cmd_ctrl = lpc32xx_nand_cmd_ctrl; + chip->legacy.dev_ready = lpc32xx_nand_device_ready; + chip->legacy.chip_delay = 20; /* 20us command delay time */ /* Init NAND controller */ lpc32xx_nand_setup(host); @@ -891,9 +891,9 @@ static int lpc32xx_nand_probe(struct platform_device *pdev) /* NAND callbacks for LPC32xx SLC hardware */ chip->ecc.mode = NAND_ECC_HW_SYNDROME; - chip->read_byte = lpc32xx_nand_read_byte; - chip->read_buf = lpc32xx_nand_read_buf; - chip->write_buf = lpc32xx_nand_write_buf; + chip->legacy.read_byte = lpc32xx_nand_read_byte; + chip->legacy.read_buf = lpc32xx_nand_read_buf; + chip->legacy.write_buf = lpc32xx_nand_write_buf; chip->ecc.read_page_raw = lpc32xx_nand_read_page_raw_syndrome; chip->ecc.read_page = lpc32xx_nand_read_page_syndrome; chip->ecc.write_page_raw = lpc32xx_nand_write_page_raw_syndrome; @@ -925,7 +925,7 @@ static int lpc32xx_nand_probe(struct platform_device *pdev) /* Find NAND device */ chip->dummy_controller.ops = &lpc32xx_nand_controller_ops; - res = nand_scan(mtd, 1); + res = nand_scan(chip, 1); if (res) goto release_dma; @@ -956,9 +956,8 @@ static int lpc32xx_nand_remove(struct platform_device *pdev) { uint32_t tmp; struct lpc32xx_nand_host *host = platform_get_drvdata(pdev); - struct mtd_info *mtd = nand_to_mtd(&host->nand_chip); - nand_release(mtd); + nand_release(&host->nand_chip); dma_release_channel(host->dma_chan); /* Force CE high */ diff --git a/drivers/mtd/nand/raw/marvell_nand.c b/drivers/mtd/nand/raw/marvell_nand.c index bc2ef5209783..650f2b490a05 100644 --- a/drivers/mtd/nand/raw/marvell_nand.c +++ b/drivers/mtd/nand/raw/marvell_nand.c @@ -5,6 +5,73 @@ * Copyright (C) 2017 Marvell * Author: Miquel RAYNAL <miquel.raynal@free-electrons.com> * + * + * This NAND controller driver handles two versions of the hardware, + * one is called NFCv1 and is available on PXA SoCs and the other is + * called NFCv2 and is available on Armada SoCs. + * + * The main visible difference is that NFCv1 only has Hamming ECC + * capabilities, while NFCv2 also embeds a BCH ECC engine. Also, DMA + * is not used with NFCv2. + * + * The ECC layouts are depicted in details in Marvell AN-379, but here + * is a brief description. + * + * When using Hamming, the data is split in 512B chunks (either 1, 2 + * or 4) and each chunk will have its own ECC "digest" of 6B at the + * beginning of the OOB area and eventually the remaining free OOB + * bytes (also called "spare" bytes in the driver). This engine + * corrects up to 1 bit per chunk and detects reliably an error if + * there are at most 2 bitflips. Here is the page layout used by the + * controller when Hamming is chosen: + * + * +-------------------------------------------------------------+ + * | Data 1 | ... | Data N | ECC 1 | ... | ECCN | Free OOB bytes | + * +-------------------------------------------------------------+ + * + * When using the BCH engine, there are N identical (data + free OOB + + * ECC) sections and potentially an extra one to deal with + * configurations where the chosen (data + free OOB + ECC) sizes do + * not align with the page (data + OOB) size. ECC bytes are always + * 30B per ECC chunk. Here is the page layout used by the controller + * when BCH is chosen: + * + * +----------------------------------------- + * | Data 1 | Free OOB bytes 1 | ECC 1 | ... + * +----------------------------------------- + * + * ------------------------------------------- + * ... | Data N | Free OOB bytes N | ECC N | + * ------------------------------------------- + * + * --------------------------------------------+ + * Last Data | Last Free OOB bytes | Last ECC | + * --------------------------------------------+ + * + * In both cases, the layout seen by the user is always: all data + * first, then all free OOB bytes and finally all ECC bytes. With BCH, + * ECC bytes are 30B long and are padded with 0xFF to align on 32 + * bytes. + * + * The controller has certain limitations that are handled by the + * driver: + * - It can only read 2k at a time. To overcome this limitation, the + * driver issues data cycles on the bus, without issuing new + * CMD + ADDR cycles. The Marvell term is "naked" operations. + * - The ECC strength in BCH mode cannot be tuned. It is fixed 16 + * bits. What can be tuned is the ECC block size as long as it + * stays between 512B and 2kiB. It's usually chosen based on the + * chip ECC requirements. For instance, using 2kiB ECC chunks + * provides 4b/512B correctability. + * - The controller will always treat data bytes, free OOB bytes + * and ECC bytes in that order, no matter what the real layout is + * (which is usually all data then all OOB bytes). The + * marvell_nfc_layouts array below contains the currently + * supported layouts. + * - Because of these weird layouts, the Bad Block Markers can be + * located in data section. In this case, the NAND_BBT_NO_OOB_BBM + * option must be set to prevent scanning/writing bad block + * markers. */ #include <linux/module.h> @@ -217,8 +284,11 @@ static const struct marvell_hw_ecc_layout marvell_nfc_layouts[] = { MARVELL_LAYOUT( 512, 512, 1, 1, 1, 512, 8, 8, 0, 0, 0), MARVELL_LAYOUT( 2048, 512, 1, 1, 1, 2048, 40, 24, 0, 0, 0), MARVELL_LAYOUT( 2048, 512, 4, 1, 1, 2048, 32, 30, 0, 0, 0), + MARVELL_LAYOUT( 2048, 512, 8, 2, 1, 1024, 0, 30,1024,32, 30), MARVELL_LAYOUT( 4096, 512, 4, 2, 2, 2048, 32, 30, 0, 0, 0), MARVELL_LAYOUT( 4096, 512, 8, 5, 4, 1024, 0, 30, 0, 64, 30), + MARVELL_LAYOUT( 8192, 512, 4, 4, 4, 2048, 0, 30, 0, 0, 0), + MARVELL_LAYOUT( 8192, 512, 8, 9, 8, 1024, 0, 30, 0, 160, 30), }; /** @@ -634,9 +704,8 @@ static int marvell_nfc_wait_op(struct nand_chip *chip, unsigned int timeout_ms) return 0; } -static void marvell_nfc_select_chip(struct mtd_info *mtd, int die_nr) +static void marvell_nfc_select_chip(struct nand_chip *chip, int die_nr) { - struct nand_chip *chip = mtd_to_nand(mtd); struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip); struct marvell_nfc *nfc = to_marvell_nfc(chip->controller); u32 ndcr_generic; @@ -686,7 +755,7 @@ static irqreturn_t marvell_nfc_isr(int irq, void *dev_id) marvell_nfc_disable_int(nfc, st & NDCR_ALL_INT); - if (!(st & (NDSR_RDDREQ | NDSR_WRDREQ | NDSR_WRCMDREQ))) + if (st & (NDSR_RDY(0) | NDSR_RDY(1))) complete(&nfc->complete); return IRQ_HANDLED; @@ -959,18 +1028,15 @@ static int marvell_nfc_hw_ecc_hmg_do_read_page(struct nand_chip *chip, return ret; } -static int marvell_nfc_hw_ecc_hmg_read_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, u8 *buf, +static int marvell_nfc_hw_ecc_hmg_read_page_raw(struct nand_chip *chip, u8 *buf, int oob_required, int page) { return marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi, true, page); } -static int marvell_nfc_hw_ecc_hmg_read_page(struct mtd_info *mtd, - struct nand_chip *chip, - u8 *buf, int oob_required, - int page) +static int marvell_nfc_hw_ecc_hmg_read_page(struct nand_chip *chip, u8 *buf, + int oob_required, int page) { const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout; unsigned int full_sz = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes; @@ -1008,8 +1074,7 @@ static int marvell_nfc_hw_ecc_hmg_read_page(struct mtd_info *mtd, * it appears before the ECC bytes when reading), the ->read_oob_raw() function * also stands for ->read_oob(). */ -static int marvell_nfc_hw_ecc_hmg_read_oob_raw(struct mtd_info *mtd, - struct nand_chip *chip, int page) +static int marvell_nfc_hw_ecc_hmg_read_oob_raw(struct nand_chip *chip, int page) { /* Invalidate page cache */ chip->pagebuf = -1; @@ -1073,8 +1138,7 @@ static int marvell_nfc_hw_ecc_hmg_do_write_page(struct nand_chip *chip, return ret; } -static int marvell_nfc_hw_ecc_hmg_write_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, +static int marvell_nfc_hw_ecc_hmg_write_page_raw(struct nand_chip *chip, const u8 *buf, int oob_required, int page) { @@ -1082,8 +1146,7 @@ static int marvell_nfc_hw_ecc_hmg_write_page_raw(struct mtd_info *mtd, true, page); } -static int marvell_nfc_hw_ecc_hmg_write_page(struct mtd_info *mtd, - struct nand_chip *chip, +static int marvell_nfc_hw_ecc_hmg_write_page(struct nand_chip *chip, const u8 *buf, int oob_required, int page) { @@ -1102,10 +1165,11 @@ static int marvell_nfc_hw_ecc_hmg_write_page(struct mtd_info *mtd, * it appears before the ECC bytes when reading), the ->write_oob_raw() function * also stands for ->write_oob(). */ -static int marvell_nfc_hw_ecc_hmg_write_oob_raw(struct mtd_info *mtd, - struct nand_chip *chip, +static int marvell_nfc_hw_ecc_hmg_write_oob_raw(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + /* Invalidate page cache */ chip->pagebuf = -1; @@ -1116,10 +1180,10 @@ static int marvell_nfc_hw_ecc_hmg_write_oob_raw(struct mtd_info *mtd, } /* BCH read helpers */ -static int marvell_nfc_hw_ecc_bch_read_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, u8 *buf, +static int marvell_nfc_hw_ecc_bch_read_page_raw(struct nand_chip *chip, u8 *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout; u8 *oob = chip->oob_poi; int chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes; @@ -1228,17 +1292,17 @@ static void marvell_nfc_hw_ecc_bch_read_chunk(struct nand_chip *chip, int chunk, } } -static int marvell_nfc_hw_ecc_bch_read_page(struct mtd_info *mtd, - struct nand_chip *chip, +static int marvell_nfc_hw_ecc_bch_read_page(struct nand_chip *chip, u8 *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout; - int data_len = lt->data_bytes, spare_len = lt->spare_bytes, ecc_len; - u8 *data = buf, *spare = chip->oob_poi, *ecc; + int data_len = lt->data_bytes, spare_len = lt->spare_bytes; + u8 *data = buf, *spare = chip->oob_poi; int max_bitflips = 0; u32 failure_mask = 0; - int chunk, ecc_offset_in_page, ret; + int chunk, ret; /* * With BCH, OOB is not fully used (and thus not read entirely), not @@ -1279,73 +1343,98 @@ static int marvell_nfc_hw_ecc_bch_read_page(struct mtd_info *mtd, * the controller in normal mode and must be re-read in raw mode. To * avoid dropping the performances, we prefer not to include them. The * user should re-read the page in raw mode if ECC bytes are required. + */ + + /* + * In case there is any subpage read error reported by ->correct(), we + * usually re-read only ECC bytes in raw mode and check if the whole + * page is empty. In this case, it is normal that the ECC check failed + * and we just ignore the error. * - * However, for any subpage read error reported by ->correct(), the ECC - * bytes must be read in raw mode and the full subpage must be checked - * to see if it is entirely empty of if there was an actual error. + * However, it has been empirically observed that for some layouts (e.g + * 2k page, 8b strength per 512B chunk), the controller tries to correct + * bits and may create itself bitflips in the erased area. To overcome + * this strange behavior, the whole page is re-read in raw mode, not + * only the ECC bytes. */ for (chunk = 0; chunk < lt->nchunks; chunk++) { + int data_off_in_page, spare_off_in_page, ecc_off_in_page; + int data_off, spare_off, ecc_off; + int data_len, spare_len, ecc_len; + /* No failure reported for this chunk, move to the next one */ if (!(failure_mask & BIT(chunk))) continue; - /* Derive ECC bytes positions (in page/buffer) and length */ - ecc = chip->oob_poi + - (lt->full_chunk_cnt * lt->spare_bytes) + - lt->last_spare_bytes + - (chunk * ALIGN(lt->ecc_bytes, 32)); - ecc_offset_in_page = - (chunk * (lt->data_bytes + lt->spare_bytes + - lt->ecc_bytes)) + - (chunk < lt->full_chunk_cnt ? - lt->data_bytes + lt->spare_bytes : - lt->last_data_bytes + lt->last_spare_bytes); - ecc_len = chunk < lt->full_chunk_cnt ? - lt->ecc_bytes : lt->last_ecc_bytes; - - /* Do the actual raw read of the ECC bytes */ - nand_change_read_column_op(chip, ecc_offset_in_page, - ecc, ecc_len, false); - - /* Derive data/spare bytes positions (in buffer) and length */ - data = buf + (chunk * lt->data_bytes); - data_len = chunk < lt->full_chunk_cnt ? - lt->data_bytes : lt->last_data_bytes; - spare = chip->oob_poi + (chunk * (lt->spare_bytes + - lt->ecc_bytes)); - spare_len = chunk < lt->full_chunk_cnt ? - lt->spare_bytes : lt->last_spare_bytes; + data_off_in_page = chunk * (lt->data_bytes + lt->spare_bytes + + lt->ecc_bytes); + spare_off_in_page = data_off_in_page + + (chunk < lt->full_chunk_cnt ? lt->data_bytes : + lt->last_data_bytes); + ecc_off_in_page = spare_off_in_page + + (chunk < lt->full_chunk_cnt ? lt->spare_bytes : + lt->last_spare_bytes); + + data_off = chunk * lt->data_bytes; + spare_off = chunk * lt->spare_bytes; + ecc_off = (lt->full_chunk_cnt * lt->spare_bytes) + + lt->last_spare_bytes + + (chunk * (lt->ecc_bytes + 2)); + + data_len = chunk < lt->full_chunk_cnt ? lt->data_bytes : + lt->last_data_bytes; + spare_len = chunk < lt->full_chunk_cnt ? lt->spare_bytes : + lt->last_spare_bytes; + ecc_len = chunk < lt->full_chunk_cnt ? lt->ecc_bytes : + lt->last_ecc_bytes; + + /* + * Only re-read the ECC bytes, unless we are using the 2k/8b + * layout which is buggy in the sense that the ECC engine will + * try to correct data bytes anyway, creating bitflips. In this + * case, re-read the entire page. + */ + if (lt->writesize == 2048 && lt->strength == 8) { + nand_change_read_column_op(chip, data_off_in_page, + buf + data_off, data_len, + false); + nand_change_read_column_op(chip, spare_off_in_page, + chip->oob_poi + spare_off, spare_len, + false); + } + + nand_change_read_column_op(chip, ecc_off_in_page, + chip->oob_poi + ecc_off, ecc_len, + false); /* Check the entire chunk (data + spare + ecc) for emptyness */ - marvell_nfc_check_empty_chunk(chip, data, data_len, spare, - spare_len, ecc, ecc_len, + marvell_nfc_check_empty_chunk(chip, buf + data_off, data_len, + chip->oob_poi + spare_off, spare_len, + chip->oob_poi + ecc_off, ecc_len, &max_bitflips); } return max_bitflips; } -static int marvell_nfc_hw_ecc_bch_read_oob_raw(struct mtd_info *mtd, - struct nand_chip *chip, int page) +static int marvell_nfc_hw_ecc_bch_read_oob_raw(struct nand_chip *chip, int page) { /* Invalidate page cache */ chip->pagebuf = -1; - return chip->ecc.read_page_raw(mtd, chip, chip->data_buf, true, page); + return chip->ecc.read_page_raw(chip, chip->data_buf, true, page); } -static int marvell_nfc_hw_ecc_bch_read_oob(struct mtd_info *mtd, - struct nand_chip *chip, int page) +static int marvell_nfc_hw_ecc_bch_read_oob(struct nand_chip *chip, int page) { /* Invalidate page cache */ chip->pagebuf = -1; - return chip->ecc.read_page(mtd, chip, chip->data_buf, true, page); + return chip->ecc.read_page(chip, chip->data_buf, true, page); } /* BCH write helpers */ -static int marvell_nfc_hw_ecc_bch_write_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, +static int marvell_nfc_hw_ecc_bch_write_page_raw(struct nand_chip *chip, const u8 *buf, int oob_required, int page) { @@ -1458,11 +1547,11 @@ marvell_nfc_hw_ecc_bch_write_chunk(struct nand_chip *chip, int chunk, return 0; } -static int marvell_nfc_hw_ecc_bch_write_page(struct mtd_info *mtd, - struct nand_chip *chip, +static int marvell_nfc_hw_ecc_bch_write_page(struct nand_chip *chip, const u8 *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout; const u8 *data = buf; const u8 *spare = chip->oob_poi; @@ -1507,27 +1596,29 @@ static int marvell_nfc_hw_ecc_bch_write_page(struct mtd_info *mtd, return 0; } -static int marvell_nfc_hw_ecc_bch_write_oob_raw(struct mtd_info *mtd, - struct nand_chip *chip, +static int marvell_nfc_hw_ecc_bch_write_oob_raw(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + /* Invalidate page cache */ chip->pagebuf = -1; memset(chip->data_buf, 0xFF, mtd->writesize); - return chip->ecc.write_page_raw(mtd, chip, chip->data_buf, true, page); + return chip->ecc.write_page_raw(chip, chip->data_buf, true, page); } -static int marvell_nfc_hw_ecc_bch_write_oob(struct mtd_info *mtd, - struct nand_chip *chip, int page) +static int marvell_nfc_hw_ecc_bch_write_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + /* Invalidate page cache */ chip->pagebuf = -1; memset(chip->data_buf, 0xFF, mtd->writesize); - return chip->ecc.write_page(mtd, chip, chip->data_buf, true, page); + return chip->ecc.write_page(chip, chip->data_buf, true, page); } /* NAND framework ->exec_op() hooks and related helpers */ @@ -2097,6 +2188,16 @@ static int marvell_nand_hw_ecc_ctrl_init(struct mtd_info *mtd, return -ENOTSUPP; } + /* Special care for the layout 2k/8-bit/512B */ + if (l->writesize == 2048 && l->strength == 8) { + if (mtd->oobsize < 128) { + dev_err(nfc->dev, "Requested layout needs at least 128 OOB bytes\n"); + return -ENOTSUPP; + } else { + chip->bbt_options |= NAND_BBT_NO_OOB_BBM; + } + } + mtd_set_ooblayout(mtd, &marvell_nand_ooblayout_ops); ecc->steps = l->nchunks; ecc->size = l->data_bytes; @@ -2192,11 +2293,10 @@ static struct nand_bbt_descr bbt_mirror_descr = { .pattern = bbt_mirror_pattern }; -static int marvell_nfc_setup_data_interface(struct mtd_info *mtd, int chipnr, +static int marvell_nfc_setup_data_interface(struct nand_chip *chip, int chipnr, const struct nand_data_interface *conf) { - struct nand_chip *chip = mtd_to_nand(mtd); struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip); struct marvell_nfc *nfc = to_marvell_nfc(chip->controller); unsigned int period_ns = 1000000000 / clk_get_rate(nfc->core_clk) * 2; @@ -2540,7 +2640,7 @@ static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc, chip->options |= NAND_BUSWIDTH_AUTO; - ret = nand_scan(mtd, marvell_nand->nsels); + ret = nand_scan(chip, marvell_nand->nsels); if (ret) { dev_err(dev, "could not scan the nand chip\n"); return ret; @@ -2553,7 +2653,7 @@ static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc, ret = mtd_device_register(mtd, NULL, 0); if (ret) { dev_err(dev, "failed to register mtd device: %d\n", ret); - nand_release(mtd); + nand_release(chip); return ret; } @@ -2608,7 +2708,7 @@ static void marvell_nand_chips_cleanup(struct marvell_nfc *nfc) struct marvell_nand_chip *entry, *temp; list_for_each_entry_safe(entry, temp, &nfc->chips, node) { - nand_release(nand_to_mtd(&entry->chip)); + nand_release(&entry->chip); list_del(&entry->node); } } @@ -2699,24 +2799,23 @@ static int marvell_nfc_init(struct marvell_nfc *nfc) struct regmap *sysctrl_base = syscon_regmap_lookup_by_phandle(np, "marvell,system-controller"); - u32 reg; if (IS_ERR(sysctrl_base)) return PTR_ERR(sysctrl_base); - reg = GENCONF_SOC_DEVICE_MUX_NFC_EN | - GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST | - GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST | - GENCONF_SOC_DEVICE_MUX_NFC_INT_EN; - regmap_write(sysctrl_base, GENCONF_SOC_DEVICE_MUX, reg); + regmap_write(sysctrl_base, GENCONF_SOC_DEVICE_MUX, + GENCONF_SOC_DEVICE_MUX_NFC_EN | + GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST | + GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST | + GENCONF_SOC_DEVICE_MUX_NFC_INT_EN); - regmap_read(sysctrl_base, GENCONF_CLK_GATING_CTRL, ®); - reg |= GENCONF_CLK_GATING_CTRL_ND_GATE; - regmap_write(sysctrl_base, GENCONF_CLK_GATING_CTRL, reg); + regmap_update_bits(sysctrl_base, GENCONF_CLK_GATING_CTRL, + GENCONF_CLK_GATING_CTRL_ND_GATE, + GENCONF_CLK_GATING_CTRL_ND_GATE); - regmap_read(sysctrl_base, GENCONF_ND_CLK_CTRL, ®); - reg |= GENCONF_ND_CLK_CTRL_EN; - regmap_write(sysctrl_base, GENCONF_ND_CLK_CTRL, reg); + regmap_update_bits(sysctrl_base, GENCONF_ND_CLK_CTRL, + GENCONF_ND_CLK_CTRL_EN, + GENCONF_ND_CLK_CTRL_EN); } /* Configure the DMA if appropriate */ diff --git a/drivers/mtd/nand/raw/mpc5121_nfc.c b/drivers/mtd/nand/raw/mpc5121_nfc.c index 6d1740d54e0d..86a0aabe08df 100644 --- a/drivers/mtd/nand/raw/mpc5121_nfc.c +++ b/drivers/mtd/nand/raw/mpc5121_nfc.c @@ -263,8 +263,10 @@ static void mpc5121_nfc_addr_cycle(struct mtd_info *mtd, int column, int page) } /* Control chip select signals */ -static void mpc5121_nfc_select_chip(struct mtd_info *mtd, int chip) +static void mpc5121_nfc_select_chip(struct nand_chip *nand, int chip) { + struct mtd_info *mtd = nand_to_mtd(nand); + if (chip < 0) { nfc_clear(mtd, NFC_CONFIG1, NFC_CE); return; @@ -299,9 +301,9 @@ static int ads5121_chipselect_init(struct mtd_info *mtd) } /* Control chips select signal on ADS5121 board */ -static void ads5121_select_chip(struct mtd_info *mtd, int chip) +static void ads5121_select_chip(struct nand_chip *nand, int chip) { - struct nand_chip *nand = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(nand); struct mpc5121_nfc_prv *prv = nand_get_controller_data(nand); u8 v; @@ -309,16 +311,16 @@ static void ads5121_select_chip(struct mtd_info *mtd, int chip) v |= 0x0F; if (chip >= 0) { - mpc5121_nfc_select_chip(mtd, 0); + mpc5121_nfc_select_chip(nand, 0); v &= ~(1 << chip); } else - mpc5121_nfc_select_chip(mtd, -1); + mpc5121_nfc_select_chip(nand, -1); out_8(prv->csreg, v); } /* Read NAND Ready/Busy signal */ -static int mpc5121_nfc_dev_ready(struct mtd_info *mtd) +static int mpc5121_nfc_dev_ready(struct nand_chip *nand) { /* * NFC handles ready/busy signal internally. Therefore, this function @@ -328,10 +330,10 @@ static int mpc5121_nfc_dev_ready(struct mtd_info *mtd) } /* Write command to NAND flash */ -static void mpc5121_nfc_command(struct mtd_info *mtd, unsigned command, - int column, int page) +static void mpc5121_nfc_command(struct nand_chip *chip, unsigned command, + int column, int page) { - struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(chip); struct mpc5121_nfc_prv *prv = nand_get_controller_data(chip); prv->column = (column >= 0) ? column : 0; @@ -362,7 +364,7 @@ static void mpc5121_nfc_command(struct mtd_info *mtd, unsigned command, break; case NAND_CMD_SEQIN: - mpc5121_nfc_command(mtd, NAND_CMD_READ0, column, page); + mpc5121_nfc_command(chip, NAND_CMD_READ0, column, page); column = 0; break; @@ -493,34 +495,24 @@ static void mpc5121_nfc_buf_copy(struct mtd_info *mtd, u_char *buf, int len, } /* Read data from NFC buffers */ -static void mpc5121_nfc_read_buf(struct mtd_info *mtd, u_char *buf, int len) +static void mpc5121_nfc_read_buf(struct nand_chip *chip, u_char *buf, int len) { - mpc5121_nfc_buf_copy(mtd, buf, len, 0); + mpc5121_nfc_buf_copy(nand_to_mtd(chip), buf, len, 0); } /* Write data to NFC buffers */ -static void mpc5121_nfc_write_buf(struct mtd_info *mtd, - const u_char *buf, int len) +static void mpc5121_nfc_write_buf(struct nand_chip *chip, const u_char *buf, + int len) { - mpc5121_nfc_buf_copy(mtd, (u_char *)buf, len, 1); + mpc5121_nfc_buf_copy(nand_to_mtd(chip), (u_char *)buf, len, 1); } /* Read byte from NFC buffers */ -static u8 mpc5121_nfc_read_byte(struct mtd_info *mtd) +static u8 mpc5121_nfc_read_byte(struct nand_chip *chip) { u8 tmp; - mpc5121_nfc_read_buf(mtd, &tmp, sizeof(tmp)); - - return tmp; -} - -/* Read word from NFC buffers */ -static u16 mpc5121_nfc_read_word(struct mtd_info *mtd) -{ - u16 tmp; - - mpc5121_nfc_read_buf(mtd, (u_char *)&tmp, sizeof(tmp)); + mpc5121_nfc_read_buf(chip, &tmp, sizeof(tmp)); return tmp; } @@ -700,15 +692,14 @@ static int mpc5121_nfc_probe(struct platform_device *op) } mtd->name = "MPC5121 NAND"; - chip->dev_ready = mpc5121_nfc_dev_ready; - chip->cmdfunc = mpc5121_nfc_command; - chip->read_byte = mpc5121_nfc_read_byte; - chip->read_word = mpc5121_nfc_read_word; - chip->read_buf = mpc5121_nfc_read_buf; - chip->write_buf = mpc5121_nfc_write_buf; + chip->legacy.dev_ready = mpc5121_nfc_dev_ready; + chip->legacy.cmdfunc = mpc5121_nfc_command; + chip->legacy.read_byte = mpc5121_nfc_read_byte; + chip->legacy.read_buf = mpc5121_nfc_read_buf; + chip->legacy.write_buf = mpc5121_nfc_write_buf; chip->select_chip = mpc5121_nfc_select_chip; - chip->set_features = nand_get_set_features_notsupp; - chip->get_features = nand_get_set_features_notsupp; + chip->legacy.set_features = nand_get_set_features_notsupp; + chip->legacy.get_features = nand_get_set_features_notsupp; chip->bbt_options = NAND_BBT_USE_FLASH; chip->ecc.mode = NAND_ECC_SOFT; chip->ecc.algo = NAND_ECC_HAMMING; @@ -778,7 +769,7 @@ static int mpc5121_nfc_probe(struct platform_device *op) } /* Detect NAND chips */ - retval = nand_scan(mtd, be32_to_cpup(chips_no)); + retval = nand_scan(chip, be32_to_cpup(chips_no)); if (retval) { dev_err(dev, "NAND Flash not found !\n"); goto error; @@ -828,7 +819,7 @@ static int mpc5121_nfc_remove(struct platform_device *op) struct device *dev = &op->dev; struct mtd_info *mtd = dev_get_drvdata(dev); - nand_release(mtd); + nand_release(mtd_to_nand(mtd)); mpc5121_nfc_free(dev, mtd); return 0; diff --git a/drivers/mtd/nand/raw/mtk_nand.c b/drivers/mtd/nand/raw/mtk_nand.c index 57b5ed1699e3..2bb0df1b7244 100644 --- a/drivers/mtd/nand/raw/mtk_nand.c +++ b/drivers/mtd/nand/raw/mtk_nand.c @@ -389,23 +389,22 @@ static int mtk_nfc_hw_runtime_config(struct mtd_info *mtd) return 0; } -static void mtk_nfc_select_chip(struct mtd_info *mtd, int chip) +static void mtk_nfc_select_chip(struct nand_chip *nand, int chip) { - struct nand_chip *nand = mtd_to_nand(mtd); struct mtk_nfc *nfc = nand_get_controller_data(nand); struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(nand); if (chip < 0) return; - mtk_nfc_hw_runtime_config(mtd); + mtk_nfc_hw_runtime_config(nand_to_mtd(nand)); nfi_writel(nfc, mtk_nand->sels[chip], NFI_CSEL); } -static int mtk_nfc_dev_ready(struct mtd_info *mtd) +static int mtk_nfc_dev_ready(struct nand_chip *nand) { - struct mtk_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd)); + struct mtk_nfc *nfc = nand_get_controller_data(nand); if (nfi_readl(nfc, NFI_STA) & STA_BUSY) return 0; @@ -413,9 +412,10 @@ static int mtk_nfc_dev_ready(struct mtd_info *mtd) return 1; } -static void mtk_nfc_cmd_ctrl(struct mtd_info *mtd, int dat, unsigned int ctrl) +static void mtk_nfc_cmd_ctrl(struct nand_chip *chip, int dat, + unsigned int ctrl) { - struct mtk_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd)); + struct mtk_nfc *nfc = nand_get_controller_data(chip); if (ctrl & NAND_ALE) { mtk_nfc_send_address(nfc, dat); @@ -438,9 +438,8 @@ static inline void mtk_nfc_wait_ioready(struct mtk_nfc *nfc) dev_err(nfc->dev, "data not ready\n"); } -static inline u8 mtk_nfc_read_byte(struct mtd_info *mtd) +static inline u8 mtk_nfc_read_byte(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct mtk_nfc *nfc = nand_get_controller_data(chip); u32 reg; @@ -467,17 +466,17 @@ static inline u8 mtk_nfc_read_byte(struct mtd_info *mtd) return nfi_readb(nfc, NFI_DATAR); } -static void mtk_nfc_read_buf(struct mtd_info *mtd, u8 *buf, int len) +static void mtk_nfc_read_buf(struct nand_chip *chip, u8 *buf, int len) { int i; for (i = 0; i < len; i++) - buf[i] = mtk_nfc_read_byte(mtd); + buf[i] = mtk_nfc_read_byte(chip); } -static void mtk_nfc_write_byte(struct mtd_info *mtd, u8 byte) +static void mtk_nfc_write_byte(struct nand_chip *chip, u8 byte) { - struct mtk_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd)); + struct mtk_nfc *nfc = nand_get_controller_data(chip); u32 reg; reg = nfi_readl(nfc, NFI_STA) & NFI_FSM_MASK; @@ -496,18 +495,18 @@ static void mtk_nfc_write_byte(struct mtd_info *mtd, u8 byte) nfi_writeb(nfc, byte, NFI_DATAW); } -static void mtk_nfc_write_buf(struct mtd_info *mtd, const u8 *buf, int len) +static void mtk_nfc_write_buf(struct nand_chip *chip, const u8 *buf, int len) { int i; for (i = 0; i < len; i++) - mtk_nfc_write_byte(mtd, buf[i]); + mtk_nfc_write_byte(chip, buf[i]); } -static int mtk_nfc_setup_data_interface(struct mtd_info *mtd, int csline, +static int mtk_nfc_setup_data_interface(struct nand_chip *chip, int csline, const struct nand_data_interface *conf) { - struct mtk_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd)); + struct mtk_nfc *nfc = nand_get_controller_data(chip); const struct nand_sdr_timings *timings; u32 rate, tpoecs, tprecs, tc2r, tw2r, twh, twst, trlt; @@ -807,27 +806,27 @@ static int mtk_nfc_write_page(struct mtd_info *mtd, struct nand_chip *chip, return nand_prog_page_end_op(chip); } -static int mtk_nfc_write_page_hwecc(struct mtd_info *mtd, - struct nand_chip *chip, const u8 *buf, +static int mtk_nfc_write_page_hwecc(struct nand_chip *chip, const u8 *buf, int oob_on, int page) { - return mtk_nfc_write_page(mtd, chip, buf, page, 0); + return mtk_nfc_write_page(nand_to_mtd(chip), chip, buf, page, 0); } -static int mtk_nfc_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip, - const u8 *buf, int oob_on, int pg) +static int mtk_nfc_write_page_raw(struct nand_chip *chip, const u8 *buf, + int oob_on, int pg) { + struct mtd_info *mtd = nand_to_mtd(chip); struct mtk_nfc *nfc = nand_get_controller_data(chip); mtk_nfc_format_page(mtd, buf); return mtk_nfc_write_page(mtd, chip, nfc->buffer, pg, 1); } -static int mtk_nfc_write_subpage_hwecc(struct mtd_info *mtd, - struct nand_chip *chip, u32 offset, +static int mtk_nfc_write_subpage_hwecc(struct nand_chip *chip, u32 offset, u32 data_len, const u8 *buf, int oob_on, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct mtk_nfc *nfc = nand_get_controller_data(chip); int ret; @@ -839,10 +838,9 @@ static int mtk_nfc_write_subpage_hwecc(struct mtd_info *mtd, return mtk_nfc_write_page(mtd, chip, nfc->buffer, page, 1); } -static int mtk_nfc_write_oob_std(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int mtk_nfc_write_oob_std(struct nand_chip *chip, int page) { - return mtk_nfc_write_page_raw(mtd, chip, NULL, 1, page); + return mtk_nfc_write_page_raw(chip, NULL, 1, page); } static int mtk_nfc_update_ecc_stats(struct mtd_info *mtd, u8 *buf, u32 sectors) @@ -969,23 +967,25 @@ done: return bitflips; } -static int mtk_nfc_read_subpage_hwecc(struct mtd_info *mtd, - struct nand_chip *chip, u32 off, +static int mtk_nfc_read_subpage_hwecc(struct nand_chip *chip, u32 off, u32 len, u8 *p, int pg) { - return mtk_nfc_read_subpage(mtd, chip, off, len, p, pg, 0); + return mtk_nfc_read_subpage(nand_to_mtd(chip), chip, off, len, p, pg, + 0); } -static int mtk_nfc_read_page_hwecc(struct mtd_info *mtd, - struct nand_chip *chip, u8 *p, - int oob_on, int pg) +static int mtk_nfc_read_page_hwecc(struct nand_chip *chip, u8 *p, int oob_on, + int pg) { + struct mtd_info *mtd = nand_to_mtd(chip); + return mtk_nfc_read_subpage(mtd, chip, 0, mtd->writesize, p, pg, 0); } -static int mtk_nfc_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip, - u8 *buf, int oob_on, int page) +static int mtk_nfc_read_page_raw(struct nand_chip *chip, u8 *buf, int oob_on, + int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip); struct mtk_nfc *nfc = nand_get_controller_data(chip); struct mtk_nfc_fdm *fdm = &mtk_nand->fdm; @@ -1011,10 +1011,9 @@ static int mtk_nfc_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip, return ret; } -static int mtk_nfc_read_oob_std(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int mtk_nfc_read_oob_std(struct nand_chip *chip, int page) { - return mtk_nfc_read_page_raw(mtd, chip, NULL, 1, page); + return mtk_nfc_read_page_raw(chip, NULL, 1, page); } static inline void mtk_nfc_hw_init(struct mtk_nfc *nfc) @@ -1333,13 +1332,13 @@ static int mtk_nfc_nand_chip_init(struct device *dev, struct mtk_nfc *nfc, nand_set_controller_data(nand, nfc); nand->options |= NAND_USE_BOUNCE_BUFFER | NAND_SUBPAGE_READ; - nand->dev_ready = mtk_nfc_dev_ready; + nand->legacy.dev_ready = mtk_nfc_dev_ready; nand->select_chip = mtk_nfc_select_chip; - nand->write_byte = mtk_nfc_write_byte; - nand->write_buf = mtk_nfc_write_buf; - nand->read_byte = mtk_nfc_read_byte; - nand->read_buf = mtk_nfc_read_buf; - nand->cmd_ctrl = mtk_nfc_cmd_ctrl; + nand->legacy.write_byte = mtk_nfc_write_byte; + nand->legacy.write_buf = mtk_nfc_write_buf; + nand->legacy.read_byte = mtk_nfc_read_byte; + nand->legacy.read_buf = mtk_nfc_read_buf; + nand->legacy.cmd_ctrl = mtk_nfc_cmd_ctrl; nand->setup_data_interface = mtk_nfc_setup_data_interface; /* set default mode in case dt entry is missing */ @@ -1365,14 +1364,14 @@ static int mtk_nfc_nand_chip_init(struct device *dev, struct mtk_nfc *nfc, mtk_nfc_hw_init(nfc); - ret = nand_scan(mtd, nsels); + ret = nand_scan(nand, nsels); if (ret) return ret; ret = mtd_device_register(mtd, NULL, 0); if (ret) { dev_err(dev, "mtd parse partition error\n"); - nand_release(mtd); + nand_release(nand); return ret; } @@ -1538,7 +1537,7 @@ static int mtk_nfc_remove(struct platform_device *pdev) while (!list_empty(&nfc->chips)) { chip = list_first_entry(&nfc->chips, struct mtk_nfc_nand_chip, node); - nand_release(nand_to_mtd(&chip->nand)); + nand_release(&chip->nand); list_del(&chip->node); } diff --git a/drivers/mtd/nand/raw/mxc_nand.c b/drivers/mtd/nand/raw/mxc_nand.c index 4c9214dea424..88bd3f6a499c 100644 --- a/drivers/mtd/nand/raw/mxc_nand.c +++ b/drivers/mtd/nand/raw/mxc_nand.c @@ -136,8 +136,8 @@ struct mxc_nand_devtype_data { void (*irq_control)(struct mxc_nand_host *, int); u32 (*get_ecc_status)(struct mxc_nand_host *); const struct mtd_ooblayout_ops *ooblayout; - void (*select_chip)(struct mtd_info *mtd, int chip); - int (*setup_data_interface)(struct mtd_info *mtd, int csline, + void (*select_chip)(struct nand_chip *chip, int cs); + int (*setup_data_interface)(struct nand_chip *chip, int csline, const struct nand_data_interface *conf); void (*enable_hwecc)(struct nand_chip *chip, bool enable); @@ -701,7 +701,7 @@ static void mxc_nand_enable_hwecc_v3(struct nand_chip *chip, bool enable) } /* This functions is used by upper layer to checks if device is ready */ -static int mxc_nand_dev_ready(struct mtd_info *mtd) +static int mxc_nand_dev_ready(struct nand_chip *chip) { /* * NFC handles R/B internally. Therefore, this function @@ -816,8 +816,8 @@ static int mxc_nand_read_page_v2_v3(struct nand_chip *chip, void *buf, return max_bitflips; } -static int mxc_nand_read_page(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int mxc_nand_read_page(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { struct mxc_nand_host *host = nand_get_controller_data(chip); void *oob_buf; @@ -830,8 +830,8 @@ static int mxc_nand_read_page(struct mtd_info *mtd, struct nand_chip *chip, return host->devtype_data->read_page(chip, buf, oob_buf, 1, page); } -static int mxc_nand_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int mxc_nand_read_page_raw(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { struct mxc_nand_host *host = nand_get_controller_data(chip); void *oob_buf; @@ -844,8 +844,7 @@ static int mxc_nand_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip, return host->devtype_data->read_page(chip, buf, oob_buf, 0, page); } -static int mxc_nand_read_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int mxc_nand_read_oob(struct nand_chip *chip, int page) { struct mxc_nand_host *host = nand_get_controller_data(chip); @@ -874,22 +873,21 @@ static int mxc_nand_write_page(struct nand_chip *chip, const uint8_t *buf, return 0; } -static int mxc_nand_write_page_ecc(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, - int page) +static int mxc_nand_write_page_ecc(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { return mxc_nand_write_page(chip, buf, true, page); } -static int mxc_nand_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, int page) +static int mxc_nand_write_page_raw(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { return mxc_nand_write_page(chip, buf, false, page); } -static int mxc_nand_write_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int mxc_nand_write_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct mxc_nand_host *host = nand_get_controller_data(chip); memset(host->data_buf, 0xff, mtd->writesize); @@ -897,9 +895,8 @@ static int mxc_nand_write_oob(struct mtd_info *mtd, struct nand_chip *chip, return mxc_nand_write_page(chip, host->data_buf, false, page); } -static u_char mxc_nand_read_byte(struct mtd_info *mtd) +static u_char mxc_nand_read_byte(struct nand_chip *nand_chip) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); struct mxc_nand_host *host = nand_get_controller_data(nand_chip); uint8_t ret; @@ -921,25 +918,13 @@ static u_char mxc_nand_read_byte(struct mtd_info *mtd) return ret; } -static uint16_t mxc_nand_read_word(struct mtd_info *mtd) -{ - struct nand_chip *nand_chip = mtd_to_nand(mtd); - struct mxc_nand_host *host = nand_get_controller_data(nand_chip); - uint16_t ret; - - ret = *(uint16_t *)(host->data_buf + host->buf_start); - host->buf_start += 2; - - return ret; -} - /* Write data of length len to buffer buf. The data to be * written on NAND Flash is first copied to RAMbuffer. After the Data Input * Operation by the NFC, the data is written to NAND Flash */ -static void mxc_nand_write_buf(struct mtd_info *mtd, - const u_char *buf, int len) +static void mxc_nand_write_buf(struct nand_chip *nand_chip, const u_char *buf, + int len) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(nand_chip); struct mxc_nand_host *host = nand_get_controller_data(nand_chip); u16 col = host->buf_start; int n = mtd->oobsize + mtd->writesize - col; @@ -955,9 +940,10 @@ static void mxc_nand_write_buf(struct mtd_info *mtd, * Flash first the data output cycle is initiated by the NFC, which copies * the data to RAMbuffer. This data of length len is then copied to buffer buf. */ -static void mxc_nand_read_buf(struct mtd_info *mtd, u_char *buf, int len) +static void mxc_nand_read_buf(struct nand_chip *nand_chip, u_char *buf, + int len) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(nand_chip); struct mxc_nand_host *host = nand_get_controller_data(nand_chip); u16 col = host->buf_start; int n = mtd->oobsize + mtd->writesize - col; @@ -971,9 +957,8 @@ static void mxc_nand_read_buf(struct mtd_info *mtd, u_char *buf, int len) /* This function is used by upper layer for select and * deselect of the NAND chip */ -static void mxc_nand_select_chip_v1_v3(struct mtd_info *mtd, int chip) +static void mxc_nand_select_chip_v1_v3(struct nand_chip *nand_chip, int chip) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); struct mxc_nand_host *host = nand_get_controller_data(nand_chip); if (chip == -1) { @@ -992,9 +977,8 @@ static void mxc_nand_select_chip_v1_v3(struct mtd_info *mtd, int chip) } } -static void mxc_nand_select_chip_v2(struct mtd_info *mtd, int chip) +static void mxc_nand_select_chip_v2(struct nand_chip *nand_chip, int chip) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); struct mxc_nand_host *host = nand_get_controller_data(nand_chip); if (chip == -1) { @@ -1155,11 +1139,10 @@ static void preset_v1(struct mtd_info *mtd) writew(0x4, NFC_V1_V2_WRPROT); } -static int mxc_nand_v2_setup_data_interface(struct mtd_info *mtd, int csline, +static int mxc_nand_v2_setup_data_interface(struct nand_chip *chip, int csline, const struct nand_data_interface *conf) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); - struct mxc_nand_host *host = nand_get_controller_data(nand_chip); + struct mxc_nand_host *host = nand_get_controller_data(chip); int tRC_min_ns, tRC_ps, ret; unsigned long rate, rate_round; const struct nand_sdr_timings *timings; @@ -1349,10 +1332,10 @@ static void preset_v3(struct mtd_info *mtd) /* Used by the upper layer to write command to NAND Flash for * different operations to be carried out on NAND Flash */ -static void mxc_nand_command(struct mtd_info *mtd, unsigned command, - int column, int page_addr) +static void mxc_nand_command(struct nand_chip *nand_chip, unsigned command, + int column, int page_addr) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(nand_chip); struct mxc_nand_host *host = nand_get_controller_data(nand_chip); dev_dbg(host->dev, "mxc_nand_command (cmd = 0x%x, col = 0x%x, page = 0x%x)\n", @@ -1409,17 +1392,17 @@ static void mxc_nand_command(struct mtd_info *mtd, unsigned command, } } -static int mxc_nand_set_features(struct mtd_info *mtd, struct nand_chip *chip, - int addr, u8 *subfeature_param) +static int mxc_nand_set_features(struct nand_chip *chip, int addr, + u8 *subfeature_param) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); - struct mxc_nand_host *host = nand_get_controller_data(nand_chip); + struct mtd_info *mtd = nand_to_mtd(chip); + struct mxc_nand_host *host = nand_get_controller_data(chip); int i; host->buf_start = 0; for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i) - chip->write_byte(mtd, subfeature_param[i]); + chip->legacy.write_byte(chip, subfeature_param[i]); memcpy32_toio(host->main_area0, host->data_buf, mtd->writesize); host->devtype_data->send_cmd(host, NAND_CMD_SET_FEATURES, false); @@ -1429,11 +1412,11 @@ static int mxc_nand_set_features(struct mtd_info *mtd, struct nand_chip *chip, return 0; } -static int mxc_nand_get_features(struct mtd_info *mtd, struct nand_chip *chip, - int addr, u8 *subfeature_param) +static int mxc_nand_get_features(struct nand_chip *chip, int addr, + u8 *subfeature_param) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); - struct mxc_nand_host *host = nand_get_controller_data(nand_chip); + struct mtd_info *mtd = nand_to_mtd(chip); + struct mxc_nand_host *host = nand_get_controller_data(chip); int i; host->devtype_data->send_cmd(host, NAND_CMD_GET_FEATURES, false); @@ -1443,7 +1426,7 @@ static int mxc_nand_get_features(struct mtd_info *mtd, struct nand_chip *chip, host->buf_start = 0; for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i) - *subfeature_param++ = chip->read_byte(mtd); + *subfeature_param++ = chip->legacy.read_byte(chip); return 0; } @@ -1786,18 +1769,17 @@ static int mxcnd_probe(struct platform_device *pdev) mtd->name = DRIVER_NAME; /* 50 us command delay time */ - this->chip_delay = 5; + this->legacy.chip_delay = 5; nand_set_controller_data(this, host); nand_set_flash_node(this, pdev->dev.of_node), - this->dev_ready = mxc_nand_dev_ready; - this->cmdfunc = mxc_nand_command; - this->read_byte = mxc_nand_read_byte; - this->read_word = mxc_nand_read_word; - this->write_buf = mxc_nand_write_buf; - this->read_buf = mxc_nand_read_buf; - this->set_features = mxc_nand_set_features; - this->get_features = mxc_nand_get_features; + this->legacy.dev_ready = mxc_nand_dev_ready; + this->legacy.cmdfunc = mxc_nand_command; + this->legacy.read_byte = mxc_nand_read_byte; + this->legacy.write_buf = mxc_nand_write_buf; + this->legacy.read_buf = mxc_nand_read_buf; + this->legacy.set_features = mxc_nand_set_features; + this->legacy.get_features = mxc_nand_get_features; host->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(host->clk)) @@ -1900,7 +1882,7 @@ static int mxcnd_probe(struct platform_device *pdev) /* Scan the NAND device */ this->dummy_controller.ops = &mxcnd_controller_ops; - err = nand_scan(mtd, is_imx25_nfc(host) ? 4 : 1); + err = nand_scan(this, is_imx25_nfc(host) ? 4 : 1); if (err) goto escan; @@ -1928,7 +1910,7 @@ static int mxcnd_remove(struct platform_device *pdev) { struct mxc_nand_host *host = platform_get_drvdata(pdev); - nand_release(nand_to_mtd(&host->nand)); + nand_release(&host->nand); if (host->clk_act) clk_disable_unprepare(host->clk); diff --git a/drivers/mtd/nand/raw/nand_amd.c b/drivers/mtd/nand/raw/nand_amd.c index 22f060f38123..890c5b43e03c 100644 --- a/drivers/mtd/nand/raw/nand_amd.c +++ b/drivers/mtd/nand/raw/nand_amd.c @@ -15,7 +15,7 @@ * GNU General Public License for more details. */ -#include <linux/mtd/rawnand.h> +#include "internals.h" static void amd_nand_decode_id(struct nand_chip *chip) { diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c index d527e448ce19..05bd0779fe9b 100644 --- a/drivers/mtd/nand/raw/nand_base.c +++ b/drivers/mtd/nand/raw/nand_base.c @@ -36,10 +36,8 @@ #include <linux/sched.h> #include <linux/slab.h> #include <linux/mm.h> -#include <linux/nmi.h> #include <linux/types.h> #include <linux/mtd/mtd.h> -#include <linux/mtd/rawnand.h> #include <linux/mtd/nand_ecc.h> #include <linux/mtd/nand_bch.h> #include <linux/interrupt.h> @@ -48,6 +46,8 @@ #include <linux/mtd/partitions.h> #include <linux/of.h> +#include "internals.h" + static int nand_get_device(struct mtd_info *mtd, int new_state); static int nand_do_write_oob(struct mtd_info *mtd, loff_t to, @@ -253,183 +253,16 @@ static void nand_release_device(struct mtd_info *mtd) } /** - * nand_read_byte - [DEFAULT] read one byte from the chip - * @mtd: MTD device structure - * - * Default read function for 8bit buswidth - */ -static uint8_t nand_read_byte(struct mtd_info *mtd) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - return readb(chip->IO_ADDR_R); -} - -/** - * nand_read_byte16 - [DEFAULT] read one byte endianness aware from the chip - * @mtd: MTD device structure - * - * Default read function for 16bit buswidth with endianness conversion. - * - */ -static uint8_t nand_read_byte16(struct mtd_info *mtd) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - return (uint8_t) cpu_to_le16(readw(chip->IO_ADDR_R)); -} - -/** - * nand_read_word - [DEFAULT] read one word from the chip - * @mtd: MTD device structure - * - * Default read function for 16bit buswidth without endianness conversion. - */ -static u16 nand_read_word(struct mtd_info *mtd) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - return readw(chip->IO_ADDR_R); -} - -/** - * nand_select_chip - [DEFAULT] control CE line - * @mtd: MTD device structure - * @chipnr: chipnumber to select, -1 for deselect - * - * Default select function for 1 chip devices. - */ -static void nand_select_chip(struct mtd_info *mtd, int chipnr) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - - switch (chipnr) { - case -1: - chip->cmd_ctrl(mtd, NAND_CMD_NONE, 0 | NAND_CTRL_CHANGE); - break; - case 0: - break; - - default: - BUG(); - } -} - -/** - * nand_write_byte - [DEFAULT] write single byte to chip - * @mtd: MTD device structure - * @byte: value to write - * - * Default function to write a byte to I/O[7:0] - */ -static void nand_write_byte(struct mtd_info *mtd, uint8_t byte) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - - chip->write_buf(mtd, &byte, 1); -} - -/** - * nand_write_byte16 - [DEFAULT] write single byte to a chip with width 16 - * @mtd: MTD device structure - * @byte: value to write - * - * Default function to write a byte to I/O[7:0] on a 16-bit wide chip. - */ -static void nand_write_byte16(struct mtd_info *mtd, uint8_t byte) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - uint16_t word = byte; - - /* - * It's not entirely clear what should happen to I/O[15:8] when writing - * a byte. The ONFi spec (Revision 3.1; 2012-09-19, Section 2.16) reads: - * - * When the host supports a 16-bit bus width, only data is - * transferred at the 16-bit width. All address and command line - * transfers shall use only the lower 8-bits of the data bus. During - * command transfers, the host may place any value on the upper - * 8-bits of the data bus. During address transfers, the host shall - * set the upper 8-bits of the data bus to 00h. - * - * One user of the write_byte callback is nand_set_features. The - * four parameters are specified to be written to I/O[7:0], but this is - * neither an address nor a command transfer. Let's assume a 0 on the - * upper I/O lines is OK. - */ - chip->write_buf(mtd, (uint8_t *)&word, 2); -} - -/** - * nand_write_buf - [DEFAULT] write buffer to chip - * @mtd: MTD device structure - * @buf: data buffer - * @len: number of bytes to write - * - * Default write function for 8bit buswidth. - */ -static void nand_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - - iowrite8_rep(chip->IO_ADDR_W, buf, len); -} - -/** - * nand_read_buf - [DEFAULT] read chip data into buffer - * @mtd: MTD device structure - * @buf: buffer to store date - * @len: number of bytes to read - * - * Default read function for 8bit buswidth. - */ -static void nand_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - - ioread8_rep(chip->IO_ADDR_R, buf, len); -} - -/** - * nand_write_buf16 - [DEFAULT] write buffer to chip - * @mtd: MTD device structure - * @buf: data buffer - * @len: number of bytes to write - * - * Default write function for 16bit buswidth. - */ -static void nand_write_buf16(struct mtd_info *mtd, const uint8_t *buf, int len) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - u16 *p = (u16 *) buf; - - iowrite16_rep(chip->IO_ADDR_W, p, len >> 1); -} - -/** - * nand_read_buf16 - [DEFAULT] read chip data into buffer - * @mtd: MTD device structure - * @buf: buffer to store date - * @len: number of bytes to read - * - * Default read function for 16bit buswidth. - */ -static void nand_read_buf16(struct mtd_info *mtd, uint8_t *buf, int len) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - u16 *p = (u16 *) buf; - - ioread16_rep(chip->IO_ADDR_R, p, len >> 1); -} - -/** * nand_block_bad - [DEFAULT] Read bad block marker from the chip - * @mtd: MTD device structure + * @chip: NAND chip object * @ofs: offset from device start * * Check, if the block is bad. */ -static int nand_block_bad(struct mtd_info *mtd, loff_t ofs) +static int nand_block_bad(struct nand_chip *chip, loff_t ofs) { + struct mtd_info *mtd = nand_to_mtd(chip); int page, page_end, res; - struct nand_chip *chip = mtd_to_nand(mtd); u8 bad; if (chip->bbt_options & NAND_BBT_SCANLASTPAGE) @@ -439,7 +272,7 @@ static int nand_block_bad(struct mtd_info *mtd, loff_t ofs) page_end = page + (chip->bbt_options & NAND_BBT_SCAN2NDPAGE ? 2 : 1); for (; page < page_end; page++) { - res = chip->ecc.read_oob(mtd, chip, page); + res = chip->ecc.read_oob(chip, page); if (res < 0) return res; @@ -458,16 +291,16 @@ static int nand_block_bad(struct mtd_info *mtd, loff_t ofs) /** * nand_default_block_markbad - [DEFAULT] mark a block bad via bad block marker - * @mtd: MTD device structure + * @chip: NAND chip object * @ofs: offset from device start * * This is the default implementation, which can be overridden by a hardware * specific driver. It provides the details for writing a bad block marker to a * block. */ -static int nand_default_block_markbad(struct mtd_info *mtd, loff_t ofs) +static int nand_default_block_markbad(struct nand_chip *chip, loff_t ofs) { - struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(chip); struct mtd_oob_ops ops; uint8_t buf[2] = { 0, 0 }; int ret = 0, res, i = 0; @@ -499,13 +332,34 @@ static int nand_default_block_markbad(struct mtd_info *mtd, loff_t ofs) } /** + * nand_markbad_bbm - mark a block by updating the BBM + * @chip: NAND chip object + * @ofs: offset of the block to mark bad + */ +int nand_markbad_bbm(struct nand_chip *chip, loff_t ofs) +{ + if (chip->legacy.block_markbad) + return chip->legacy.block_markbad(chip, ofs); + + return nand_default_block_markbad(chip, ofs); +} + +static int nand_isbad_bbm(struct nand_chip *chip, loff_t ofs) +{ + if (chip->legacy.block_bad) + return chip->legacy.block_bad(chip, ofs); + + return nand_block_bad(chip, ofs); +} + +/** * nand_block_markbad_lowlevel - mark a block bad * @mtd: MTD device structure * @ofs: offset from device start * * This function performs the generic NAND bad block marking steps (i.e., bad * block table(s) and/or marker(s)). We only allow the hardware driver to - * specify how to write bad block markers to OOB (chip->block_markbad). + * specify how to write bad block markers to OOB (chip->legacy.block_markbad). * * We try operations in the following order: * @@ -529,17 +383,17 @@ static int nand_block_markbad_lowlevel(struct mtd_info *mtd, loff_t ofs) memset(&einfo, 0, sizeof(einfo)); einfo.addr = ofs; einfo.len = 1ULL << chip->phys_erase_shift; - nand_erase_nand(mtd, &einfo, 0); + nand_erase_nand(chip, &einfo, 0); /* Write bad block marker to OOB */ nand_get_device(mtd, FL_WRITING); - ret = chip->block_markbad(mtd, ofs); + ret = nand_markbad_bbm(chip, ofs); nand_release_device(mtd); } /* Mark block bad in BBT */ if (chip->bbt) { - res = nand_markbad_bbt(mtd, ofs); + res = nand_markbad_bbt(chip, ofs); if (!ret) ret = res; } @@ -589,7 +443,7 @@ static int nand_block_isreserved(struct mtd_info *mtd, loff_t ofs) if (!chip->bbt) return 0; /* Return info from the table */ - return nand_isreserved_bbt(mtd, ofs); + return nand_isreserved_bbt(chip, ofs); } /** @@ -605,89 +459,14 @@ static int nand_block_checkbad(struct mtd_info *mtd, loff_t ofs, int allowbbt) { struct nand_chip *chip = mtd_to_nand(mtd); - if (!chip->bbt) - return chip->block_bad(mtd, ofs); - /* Return info from the table */ - return nand_isbad_bbt(mtd, ofs, allowbbt); -} + if (chip->bbt) + return nand_isbad_bbt(chip, ofs, allowbbt); -/** - * panic_nand_wait_ready - [GENERIC] Wait for the ready pin after commands. - * @mtd: MTD device structure - * @timeo: Timeout - * - * Helper function for nand_wait_ready used when needing to wait in interrupt - * context. - */ -static void panic_nand_wait_ready(struct mtd_info *mtd, unsigned long timeo) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - int i; - - /* Wait for the device to get ready */ - for (i = 0; i < timeo; i++) { - if (chip->dev_ready(mtd)) - break; - touch_softlockup_watchdog(); - mdelay(1); - } + return nand_isbad_bbm(chip, ofs); } /** - * nand_wait_ready - [GENERIC] Wait for the ready pin after commands. - * @mtd: MTD device structure - * - * Wait for the ready pin after a command, and warn if a timeout occurs. - */ -void nand_wait_ready(struct mtd_info *mtd) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - unsigned long timeo = 400; - - if (in_interrupt() || oops_in_progress) - return panic_nand_wait_ready(mtd, timeo); - - /* Wait until command is processed or timeout occurs */ - timeo = jiffies + msecs_to_jiffies(timeo); - do { - if (chip->dev_ready(mtd)) - return; - cond_resched(); - } while (time_before(jiffies, timeo)); - - if (!chip->dev_ready(mtd)) - pr_warn_ratelimited("timeout while waiting for chip to become ready\n"); -} -EXPORT_SYMBOL_GPL(nand_wait_ready); - -/** - * nand_wait_status_ready - [GENERIC] Wait for the ready status after commands. - * @mtd: MTD device structure - * @timeo: Timeout in ms - * - * Wait for status ready (i.e. command done) or timeout. - */ -static void nand_wait_status_ready(struct mtd_info *mtd, unsigned long timeo) -{ - register struct nand_chip *chip = mtd_to_nand(mtd); - int ret; - - timeo = jiffies + msecs_to_jiffies(timeo); - do { - u8 status; - - ret = nand_read_data_op(chip, &status, sizeof(status), true); - if (ret) - return; - - if (status & NAND_STATUS_READY) - break; - touch_softlockup_watchdog(); - } while (time_before(jiffies, timeo)); -}; - -/** * nand_soft_waitrdy - Poll STATUS reg until RDY bit is set to 1 * @chip: NAND chip structure * @timeout_ms: Timeout in ms @@ -753,273 +532,6 @@ int nand_soft_waitrdy(struct nand_chip *chip, unsigned long timeout_ms) EXPORT_SYMBOL_GPL(nand_soft_waitrdy); /** - * nand_command - [DEFAULT] Send command to NAND device - * @mtd: MTD device structure - * @command: the command to be sent - * @column: the column address for this command, -1 if none - * @page_addr: the page address for this command, -1 if none - * - * Send command to NAND device. This function is used for small page devices - * (512 Bytes per page). - */ -static void nand_command(struct mtd_info *mtd, unsigned int command, - int column, int page_addr) -{ - register struct nand_chip *chip = mtd_to_nand(mtd); - int ctrl = NAND_CTRL_CLE | NAND_CTRL_CHANGE; - - /* Write out the command to the device */ - if (command == NAND_CMD_SEQIN) { - int readcmd; - - if (column >= mtd->writesize) { - /* OOB area */ - column -= mtd->writesize; - readcmd = NAND_CMD_READOOB; - } else if (column < 256) { - /* First 256 bytes --> READ0 */ - readcmd = NAND_CMD_READ0; - } else { - column -= 256; - readcmd = NAND_CMD_READ1; - } - chip->cmd_ctrl(mtd, readcmd, ctrl); - ctrl &= ~NAND_CTRL_CHANGE; - } - if (command != NAND_CMD_NONE) - chip->cmd_ctrl(mtd, command, ctrl); - - /* Address cycle, when necessary */ - ctrl = NAND_CTRL_ALE | NAND_CTRL_CHANGE; - /* Serially input address */ - if (column != -1) { - /* Adjust columns for 16 bit buswidth */ - if (chip->options & NAND_BUSWIDTH_16 && - !nand_opcode_8bits(command)) - column >>= 1; - chip->cmd_ctrl(mtd, column, ctrl); - ctrl &= ~NAND_CTRL_CHANGE; - } - if (page_addr != -1) { - chip->cmd_ctrl(mtd, page_addr, ctrl); - ctrl &= ~NAND_CTRL_CHANGE; - chip->cmd_ctrl(mtd, page_addr >> 8, ctrl); - if (chip->options & NAND_ROW_ADDR_3) - chip->cmd_ctrl(mtd, page_addr >> 16, ctrl); - } - chip->cmd_ctrl(mtd, NAND_CMD_NONE, NAND_NCE | NAND_CTRL_CHANGE); - - /* - * Program and erase have their own busy handlers status and sequential - * in needs no delay - */ - switch (command) { - - case NAND_CMD_NONE: - case NAND_CMD_PAGEPROG: - case NAND_CMD_ERASE1: - case NAND_CMD_ERASE2: - case NAND_CMD_SEQIN: - case NAND_CMD_STATUS: - case NAND_CMD_READID: - case NAND_CMD_SET_FEATURES: - return; - - case NAND_CMD_RESET: - if (chip->dev_ready) - break; - udelay(chip->chip_delay); - chip->cmd_ctrl(mtd, NAND_CMD_STATUS, - NAND_CTRL_CLE | NAND_CTRL_CHANGE); - chip->cmd_ctrl(mtd, - NAND_CMD_NONE, NAND_NCE | NAND_CTRL_CHANGE); - /* EZ-NAND can take upto 250ms as per ONFi v4.0 */ - nand_wait_status_ready(mtd, 250); - return; - - /* This applies to read commands */ - case NAND_CMD_READ0: - /* - * READ0 is sometimes used to exit GET STATUS mode. When this - * is the case no address cycles are requested, and we can use - * this information to detect that we should not wait for the - * device to be ready. - */ - if (column == -1 && page_addr == -1) - return; - - default: - /* - * If we don't have access to the busy pin, we apply the given - * command delay - */ - if (!chip->dev_ready) { - udelay(chip->chip_delay); - return; - } - } - /* - * Apply this short delay always to ensure that we do wait tWB in - * any case on any machine. - */ - ndelay(100); - - nand_wait_ready(mtd); -} - -static void nand_ccs_delay(struct nand_chip *chip) -{ - /* - * The controller already takes care of waiting for tCCS when the RNDIN - * or RNDOUT command is sent, return directly. - */ - if (!(chip->options & NAND_WAIT_TCCS)) - return; - - /* - * Wait tCCS_min if it is correctly defined, otherwise wait 500ns - * (which should be safe for all NANDs). - */ - if (chip->setup_data_interface) - ndelay(chip->data_interface.timings.sdr.tCCS_min / 1000); - else - ndelay(500); -} - -/** - * nand_command_lp - [DEFAULT] Send command to NAND large page device - * @mtd: MTD device structure - * @command: the command to be sent - * @column: the column address for this command, -1 if none - * @page_addr: the page address for this command, -1 if none - * - * Send command to NAND device. This is the version for the new large page - * devices. We don't have the separate regions as we have in the small page - * devices. We must emulate NAND_CMD_READOOB to keep the code compatible. - */ -static void nand_command_lp(struct mtd_info *mtd, unsigned int command, - int column, int page_addr) -{ - register struct nand_chip *chip = mtd_to_nand(mtd); - - /* Emulate NAND_CMD_READOOB */ - if (command == NAND_CMD_READOOB) { - column += mtd->writesize; - command = NAND_CMD_READ0; - } - - /* Command latch cycle */ - if (command != NAND_CMD_NONE) - chip->cmd_ctrl(mtd, command, - NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE); - - if (column != -1 || page_addr != -1) { - int ctrl = NAND_CTRL_CHANGE | NAND_NCE | NAND_ALE; - - /* Serially input address */ - if (column != -1) { - /* Adjust columns for 16 bit buswidth */ - if (chip->options & NAND_BUSWIDTH_16 && - !nand_opcode_8bits(command)) - column >>= 1; - chip->cmd_ctrl(mtd, column, ctrl); - ctrl &= ~NAND_CTRL_CHANGE; - - /* Only output a single addr cycle for 8bits opcodes. */ - if (!nand_opcode_8bits(command)) - chip->cmd_ctrl(mtd, column >> 8, ctrl); - } - if (page_addr != -1) { - chip->cmd_ctrl(mtd, page_addr, ctrl); - chip->cmd_ctrl(mtd, page_addr >> 8, - NAND_NCE | NAND_ALE); - if (chip->options & NAND_ROW_ADDR_3) - chip->cmd_ctrl(mtd, page_addr >> 16, - NAND_NCE | NAND_ALE); - } - } - chip->cmd_ctrl(mtd, NAND_CMD_NONE, NAND_NCE | NAND_CTRL_CHANGE); - - /* - * Program and erase have their own busy handlers status, sequential - * in and status need no delay. - */ - switch (command) { - - case NAND_CMD_NONE: - case NAND_CMD_CACHEDPROG: - case NAND_CMD_PAGEPROG: - case NAND_CMD_ERASE1: - case NAND_CMD_ERASE2: - case NAND_CMD_SEQIN: - case NAND_CMD_STATUS: - case NAND_CMD_READID: - case NAND_CMD_SET_FEATURES: - return; - - case NAND_CMD_RNDIN: - nand_ccs_delay(chip); - return; - - case NAND_CMD_RESET: - if (chip->dev_ready) - break; - udelay(chip->chip_delay); - chip->cmd_ctrl(mtd, NAND_CMD_STATUS, - NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE); - chip->cmd_ctrl(mtd, NAND_CMD_NONE, - NAND_NCE | NAND_CTRL_CHANGE); - /* EZ-NAND can take upto 250ms as per ONFi v4.0 */ - nand_wait_status_ready(mtd, 250); - return; - - case NAND_CMD_RNDOUT: - /* No ready / busy check necessary */ - chip->cmd_ctrl(mtd, NAND_CMD_RNDOUTSTART, - NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE); - chip->cmd_ctrl(mtd, NAND_CMD_NONE, - NAND_NCE | NAND_CTRL_CHANGE); - - nand_ccs_delay(chip); - return; - - case NAND_CMD_READ0: - /* - * READ0 is sometimes used to exit GET STATUS mode. When this - * is the case no address cycles are requested, and we can use - * this information to detect that READSTART should not be - * issued. - */ - if (column == -1 && page_addr == -1) - return; - - chip->cmd_ctrl(mtd, NAND_CMD_READSTART, - NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE); - chip->cmd_ctrl(mtd, NAND_CMD_NONE, - NAND_NCE | NAND_CTRL_CHANGE); - - /* This applies to read commands */ - default: - /* - * If we don't have access to the busy pin, we apply the given - * command delay. - */ - if (!chip->dev_ready) { - udelay(chip->chip_delay); - return; - } - } - - /* - * Apply this short delay always to ensure that we do wait tWB in - * any case on any machine. - */ - ndelay(100); - - nand_wait_ready(mtd); -} - -/** * panic_nand_get_device - [GENERIC] Get chip for selected access * @chip: the nand chip descriptor * @mtd: MTD device structure @@ -1086,13 +598,12 @@ retry: * we are in interrupt context. May happen when in panic and trying to write * an oops through mtdoops. */ -static void panic_nand_wait(struct mtd_info *mtd, struct nand_chip *chip, - unsigned long timeo) +void panic_nand_wait(struct nand_chip *chip, unsigned long timeo) { int i; for (i = 0; i < timeo; i++) { - if (chip->dev_ready) { - if (chip->dev_ready(mtd)) + if (chip->legacy.dev_ready) { + if (chip->legacy.dev_ready(chip)) break; } else { int ret; @@ -1110,60 +621,6 @@ static void panic_nand_wait(struct mtd_info *mtd, struct nand_chip *chip, } } -/** - * nand_wait - [DEFAULT] wait until the command is done - * @mtd: MTD device structure - * @chip: NAND chip structure - * - * Wait for command done. This applies to erase and program only. - */ -static int nand_wait(struct mtd_info *mtd, struct nand_chip *chip) -{ - - unsigned long timeo = 400; - u8 status; - int ret; - - /* - * Apply this short delay always to ensure that we do wait tWB in any - * case on any machine. - */ - ndelay(100); - - ret = nand_status_op(chip, NULL); - if (ret) - return ret; - - if (in_interrupt() || oops_in_progress) - panic_nand_wait(mtd, chip, timeo); - else { - timeo = jiffies + msecs_to_jiffies(timeo); - do { - if (chip->dev_ready) { - if (chip->dev_ready(mtd)) - break; - } else { - ret = nand_read_data_op(chip, &status, - sizeof(status), true); - if (ret) - return ret; - - if (status & NAND_STATUS_READY) - break; - } - cond_resched(); - } while (time_before(jiffies, timeo)); - } - - ret = nand_read_data_op(chip, &status, sizeof(status), true); - if (ret) - return ret; - - /* This can happen if in case of timeout or buggy dev_ready */ - WARN_ON(!(status & NAND_STATUS_READY)); - return status; -} - static bool nand_supports_get_features(struct nand_chip *chip, int addr) { return (chip->parameters.supports_set_get_features && @@ -1177,48 +634,6 @@ static bool nand_supports_set_features(struct nand_chip *chip, int addr) } /** - * nand_get_features - wrapper to perform a GET_FEATURE - * @chip: NAND chip info structure - * @addr: feature address - * @subfeature_param: the subfeature parameters, a four bytes array - * - * Returns 0 for success, a negative error otherwise. Returns -ENOTSUPP if the - * operation cannot be handled. - */ -int nand_get_features(struct nand_chip *chip, int addr, - u8 *subfeature_param) -{ - struct mtd_info *mtd = nand_to_mtd(chip); - - if (!nand_supports_get_features(chip, addr)) - return -ENOTSUPP; - - return chip->get_features(mtd, chip, addr, subfeature_param); -} -EXPORT_SYMBOL_GPL(nand_get_features); - -/** - * nand_set_features - wrapper to perform a SET_FEATURE - * @chip: NAND chip info structure - * @addr: feature address - * @subfeature_param: the subfeature parameters, a four bytes array - * - * Returns 0 for success, a negative error otherwise. Returns -ENOTSUPP if the - * operation cannot be handled. - */ -int nand_set_features(struct nand_chip *chip, int addr, - u8 *subfeature_param) -{ - struct mtd_info *mtd = nand_to_mtd(chip); - - if (!nand_supports_set_features(chip, addr)) - return -ENOTSUPP; - - return chip->set_features(mtd, chip, addr, subfeature_param); -} -EXPORT_SYMBOL_GPL(nand_set_features); - -/** * nand_reset_data_interface - Reset data interface and timings * @chip: The NAND chip * @chipnr: Internal die id @@ -1229,7 +644,6 @@ EXPORT_SYMBOL_GPL(nand_set_features); */ static int nand_reset_data_interface(struct nand_chip *chip, int chipnr) { - struct mtd_info *mtd = nand_to_mtd(chip); int ret; if (!chip->setup_data_interface) @@ -1250,7 +664,7 @@ static int nand_reset_data_interface(struct nand_chip *chip, int chipnr) */ onfi_fill_data_interface(chip, NAND_SDR_IFACE, 0); - ret = chip->setup_data_interface(mtd, chipnr, &chip->data_interface); + ret = chip->setup_data_interface(chip, chipnr, &chip->data_interface); if (ret) pr_err("Failed to configure data interface to SDR timing mode 0\n"); @@ -1272,7 +686,6 @@ static int nand_reset_data_interface(struct nand_chip *chip, int chipnr) */ static int nand_setup_data_interface(struct nand_chip *chip, int chipnr) { - struct mtd_info *mtd = nand_to_mtd(chip); u8 tmode_param[ONFI_SUBFEATURE_PARAM_LEN] = { chip->onfi_timing_mode_default, }; @@ -1283,16 +696,16 @@ static int nand_setup_data_interface(struct nand_chip *chip, int chipnr) /* Change the mode on the chip side (if supported by the NAND chip) */ if (nand_supports_set_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE)) { - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, chipnr); ret = nand_set_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE, tmode_param); - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); if (ret) return ret; } /* Change the mode on the controller side */ - ret = chip->setup_data_interface(mtd, chipnr, &chip->data_interface); + ret = chip->setup_data_interface(chip, chipnr, &chip->data_interface); if (ret) return ret; @@ -1301,10 +714,10 @@ static int nand_setup_data_interface(struct nand_chip *chip, int chipnr) return 0; memset(tmode_param, 0, ONFI_SUBFEATURE_PARAM_LEN); - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, chipnr); ret = nand_get_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE, tmode_param); - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); if (ret) goto err_reset_chip; @@ -1322,9 +735,9 @@ err_reset_chip: * timing mode. */ nand_reset_data_interface(chip, chipnr); - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, chipnr); nand_reset_op(chip); - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); return ret; } @@ -1345,7 +758,6 @@ err_reset_chip: */ static int nand_init_data_interface(struct nand_chip *chip) { - struct mtd_info *mtd = nand_to_mtd(chip); int modes, mode, ret; if (!chip->setup_data_interface) @@ -1356,15 +768,15 @@ static int nand_init_data_interface(struct nand_chip *chip) * if the NAND does not support ONFI, fallback to the default ONFI * timing mode. */ - modes = onfi_get_async_timing_mode(chip); - if (modes == ONFI_TIMING_MODE_UNKNOWN) { + if (chip->parameters.onfi) { + modes = chip->parameters.onfi->async_timing_mode; + } else { if (!chip->onfi_timing_mode_default) return 0; modes = GENMASK(chip->onfi_timing_mode_default, 0); } - for (mode = fls(modes) - 1; mode >= 0; mode--) { ret = onfi_fill_data_interface(chip, NAND_SDR_IFACE, mode); if (ret) @@ -1374,7 +786,7 @@ static int nand_init_data_interface(struct nand_chip *chip) * Pass NAND_DATA_IFACE_CHECK_ONLY to only check if the * controller supports the requested timings. */ - ret = chip->setup_data_interface(mtd, + ret = chip->setup_data_interface(chip, NAND_DATA_IFACE_CHECK_ONLY, &chip->data_interface); if (!ret) { @@ -1554,9 +966,9 @@ int nand_read_page_op(struct nand_chip *chip, unsigned int page, buf, len); } - chip->cmdfunc(mtd, NAND_CMD_READ0, offset_in_page, page); + chip->legacy.cmdfunc(chip, NAND_CMD_READ0, offset_in_page, page); if (len) - chip->read_buf(mtd, buf, len); + chip->legacy.read_buf(chip, buf, len); return 0; } @@ -1574,10 +986,9 @@ EXPORT_SYMBOL_GPL(nand_read_page_op); * * Returns 0 on success, a negative error code otherwise. */ -static int nand_read_param_page_op(struct nand_chip *chip, u8 page, void *buf, - unsigned int len) +int nand_read_param_page_op(struct nand_chip *chip, u8 page, void *buf, + unsigned int len) { - struct mtd_info *mtd = nand_to_mtd(chip); unsigned int i; u8 *p = buf; @@ -1603,9 +1014,9 @@ static int nand_read_param_page_op(struct nand_chip *chip, u8 page, void *buf, return nand_exec_op(chip, &op); } - chip->cmdfunc(mtd, NAND_CMD_PARAM, page, -1); + chip->legacy.cmdfunc(chip, NAND_CMD_PARAM, page, -1); for (i = 0; i < len; i++) - p[i] = chip->read_byte(mtd); + p[i] = chip->legacy.read_byte(chip); return 0; } @@ -1666,9 +1077,9 @@ int nand_change_read_column_op(struct nand_chip *chip, return nand_exec_op(chip, &op); } - chip->cmdfunc(mtd, NAND_CMD_RNDOUT, offset_in_page, -1); + chip->legacy.cmdfunc(chip, NAND_CMD_RNDOUT, offset_in_page, -1); if (len) - chip->read_buf(mtd, buf, len); + chip->legacy.read_buf(chip, buf, len); return 0; } @@ -1703,9 +1114,9 @@ int nand_read_oob_op(struct nand_chip *chip, unsigned int page, mtd->writesize + offset_in_oob, buf, len); - chip->cmdfunc(mtd, NAND_CMD_READOOB, offset_in_oob, page); + chip->legacy.cmdfunc(chip, NAND_CMD_READOOB, offset_in_oob, page); if (len) - chip->read_buf(mtd, buf, len); + chip->legacy.read_buf(chip, buf, len); return 0; } @@ -1815,10 +1226,10 @@ int nand_prog_page_begin_op(struct nand_chip *chip, unsigned int page, return nand_exec_prog_page_op(chip, page, offset_in_page, buf, len, false); - chip->cmdfunc(mtd, NAND_CMD_SEQIN, offset_in_page, page); + chip->legacy.cmdfunc(chip, NAND_CMD_SEQIN, offset_in_page, page); if (buf) - chip->write_buf(mtd, buf, len); + chip->legacy.write_buf(chip, buf, len); return 0; } @@ -1835,7 +1246,6 @@ EXPORT_SYMBOL_GPL(nand_prog_page_begin_op); */ int nand_prog_page_end_op(struct nand_chip *chip) { - struct mtd_info *mtd = nand_to_mtd(chip); int ret; u8 status; @@ -1857,8 +1267,8 @@ int nand_prog_page_end_op(struct nand_chip *chip) if (ret) return ret; } else { - chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1); - ret = chip->waitfunc(mtd, chip); + chip->legacy.cmdfunc(chip, NAND_CMD_PAGEPROG, -1, -1); + ret = chip->legacy.waitfunc(chip); if (ret < 0) return ret; @@ -1902,10 +1312,11 @@ int nand_prog_page_op(struct nand_chip *chip, unsigned int page, status = nand_exec_prog_page_op(chip, page, offset_in_page, buf, len, true); } else { - chip->cmdfunc(mtd, NAND_CMD_SEQIN, offset_in_page, page); - chip->write_buf(mtd, buf, len); - chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1); - status = chip->waitfunc(mtd, chip); + chip->legacy.cmdfunc(chip, NAND_CMD_SEQIN, offset_in_page, + page); + chip->legacy.write_buf(chip, buf, len); + chip->legacy.cmdfunc(chip, NAND_CMD_PAGEPROG, -1, -1); + status = chip->legacy.waitfunc(chip); } if (status & NAND_STATUS_FAIL) @@ -1970,9 +1381,9 @@ int nand_change_write_column_op(struct nand_chip *chip, return nand_exec_op(chip, &op); } - chip->cmdfunc(mtd, NAND_CMD_RNDIN, offset_in_page, -1); + chip->legacy.cmdfunc(chip, NAND_CMD_RNDIN, offset_in_page, -1); if (len) - chip->write_buf(mtd, buf, len); + chip->legacy.write_buf(chip, buf, len); return 0; } @@ -1994,7 +1405,6 @@ EXPORT_SYMBOL_GPL(nand_change_write_column_op); int nand_readid_op(struct nand_chip *chip, u8 addr, void *buf, unsigned int len) { - struct mtd_info *mtd = nand_to_mtd(chip); unsigned int i; u8 *id = buf; @@ -2018,10 +1428,10 @@ int nand_readid_op(struct nand_chip *chip, u8 addr, void *buf, return nand_exec_op(chip, &op); } - chip->cmdfunc(mtd, NAND_CMD_READID, addr, -1); + chip->legacy.cmdfunc(chip, NAND_CMD_READID, addr, -1); for (i = 0; i < len; i++) - id[i] = chip->read_byte(mtd); + id[i] = chip->legacy.read_byte(chip); return 0; } @@ -2040,8 +1450,6 @@ EXPORT_SYMBOL_GPL(nand_readid_op); */ int nand_status_op(struct nand_chip *chip, u8 *status) { - struct mtd_info *mtd = nand_to_mtd(chip); - if (chip->exec_op) { const struct nand_sdr_timings *sdr = nand_get_sdr_timings(&chip->data_interface); @@ -2058,9 +1466,9 @@ int nand_status_op(struct nand_chip *chip, u8 *status) return nand_exec_op(chip, &op); } - chip->cmdfunc(mtd, NAND_CMD_STATUS, -1, -1); + chip->legacy.cmdfunc(chip, NAND_CMD_STATUS, -1, -1); if (status) - *status = chip->read_byte(mtd); + *status = chip->legacy.read_byte(chip); return 0; } @@ -2079,8 +1487,6 @@ EXPORT_SYMBOL_GPL(nand_status_op); */ int nand_exit_status_op(struct nand_chip *chip) { - struct mtd_info *mtd = nand_to_mtd(chip); - if (chip->exec_op) { struct nand_op_instr instrs[] = { NAND_OP_CMD(NAND_CMD_READ0, 0), @@ -2090,11 +1496,10 @@ int nand_exit_status_op(struct nand_chip *chip) return nand_exec_op(chip, &op); } - chip->cmdfunc(mtd, NAND_CMD_READ0, -1, -1); + chip->legacy.cmdfunc(chip, NAND_CMD_READ0, -1, -1); return 0; } -EXPORT_SYMBOL_GPL(nand_exit_status_op); /** * nand_erase_op - Do an erase operation @@ -2109,7 +1514,6 @@ EXPORT_SYMBOL_GPL(nand_exit_status_op); */ int nand_erase_op(struct nand_chip *chip, unsigned int eraseblock) { - struct mtd_info *mtd = nand_to_mtd(chip); unsigned int page = eraseblock << (chip->phys_erase_shift - chip->page_shift); int ret; @@ -2139,10 +1543,10 @@ int nand_erase_op(struct nand_chip *chip, unsigned int eraseblock) if (ret) return ret; } else { - chip->cmdfunc(mtd, NAND_CMD_ERASE1, -1, page); - chip->cmdfunc(mtd, NAND_CMD_ERASE2, -1, -1); + chip->legacy.cmdfunc(chip, NAND_CMD_ERASE1, -1, page); + chip->legacy.cmdfunc(chip, NAND_CMD_ERASE2, -1, -1); - ret = chip->waitfunc(mtd, chip); + ret = chip->legacy.waitfunc(chip); if (ret < 0) return ret; @@ -2171,7 +1575,6 @@ EXPORT_SYMBOL_GPL(nand_erase_op); static int nand_set_features_op(struct nand_chip *chip, u8 feature, const void *data) { - struct mtd_info *mtd = nand_to_mtd(chip); const u8 *params = data; int i, ret; @@ -2190,11 +1593,11 @@ static int nand_set_features_op(struct nand_chip *chip, u8 feature, return nand_exec_op(chip, &op); } - chip->cmdfunc(mtd, NAND_CMD_SET_FEATURES, feature, -1); + chip->legacy.cmdfunc(chip, NAND_CMD_SET_FEATURES, feature, -1); for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i) - chip->write_byte(mtd, params[i]); + chip->legacy.write_byte(chip, params[i]); - ret = chip->waitfunc(mtd, chip); + ret = chip->legacy.waitfunc(chip); if (ret < 0) return ret; @@ -2219,7 +1622,6 @@ static int nand_set_features_op(struct nand_chip *chip, u8 feature, static int nand_get_features_op(struct nand_chip *chip, u8 feature, void *data) { - struct mtd_info *mtd = nand_to_mtd(chip); u8 *params = data; int i; @@ -2239,9 +1641,31 @@ static int nand_get_features_op(struct nand_chip *chip, u8 feature, return nand_exec_op(chip, &op); } - chip->cmdfunc(mtd, NAND_CMD_GET_FEATURES, feature, -1); + chip->legacy.cmdfunc(chip, NAND_CMD_GET_FEATURES, feature, -1); for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i) - params[i] = chip->read_byte(mtd); + params[i] = chip->legacy.read_byte(chip); + + return 0; +} + +static int nand_wait_rdy_op(struct nand_chip *chip, unsigned int timeout_ms, + unsigned int delay_ns) +{ + if (chip->exec_op) { + struct nand_op_instr instrs[] = { + NAND_OP_WAIT_RDY(PSEC_TO_MSEC(timeout_ms), + PSEC_TO_NSEC(delay_ns)), + }; + struct nand_operation op = NAND_OPERATION(instrs); + + return nand_exec_op(chip, &op); + } + + /* Apply delay or wait for ready/busy pin */ + if (!chip->legacy.dev_ready) + udelay(chip->legacy.chip_delay); + else + nand_wait_ready(chip); return 0; } @@ -2258,8 +1682,6 @@ static int nand_get_features_op(struct nand_chip *chip, u8 feature, */ int nand_reset_op(struct nand_chip *chip) { - struct mtd_info *mtd = nand_to_mtd(chip); - if (chip->exec_op) { const struct nand_sdr_timings *sdr = nand_get_sdr_timings(&chip->data_interface); @@ -2272,7 +1694,7 @@ int nand_reset_op(struct nand_chip *chip) return nand_exec_op(chip, &op); } - chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1); + chip->legacy.cmdfunc(chip, NAND_CMD_RESET, -1, -1); return 0; } @@ -2294,8 +1716,6 @@ EXPORT_SYMBOL_GPL(nand_reset_op); int nand_read_data_op(struct nand_chip *chip, void *buf, unsigned int len, bool force_8bit) { - struct mtd_info *mtd = nand_to_mtd(chip); - if (!len || !buf) return -EINVAL; @@ -2315,9 +1735,9 @@ int nand_read_data_op(struct nand_chip *chip, void *buf, unsigned int len, unsigned int i; for (i = 0; i < len; i++) - p[i] = chip->read_byte(mtd); + p[i] = chip->legacy.read_byte(chip); } else { - chip->read_buf(mtd, buf, len); + chip->legacy.read_buf(chip, buf, len); } return 0; @@ -2340,8 +1760,6 @@ EXPORT_SYMBOL_GPL(nand_read_data_op); int nand_write_data_op(struct nand_chip *chip, const void *buf, unsigned int len, bool force_8bit) { - struct mtd_info *mtd = nand_to_mtd(chip); - if (!len || !buf) return -EINVAL; @@ -2361,9 +1779,9 @@ int nand_write_data_op(struct nand_chip *chip, const void *buf, unsigned int i; for (i = 0; i < len; i++) - chip->write_byte(mtd, p[i]); + chip->legacy.write_byte(chip, p[i]); } else { - chip->write_buf(mtd, buf, len); + chip->legacy.write_buf(chip, buf, len); } return 0; @@ -2798,7 +2216,6 @@ EXPORT_SYMBOL_GPL(nand_subop_get_data_len); */ int nand_reset(struct nand_chip *chip, int chipnr) { - struct mtd_info *mtd = nand_to_mtd(chip); struct nand_data_interface saved_data_intf = chip->data_interface; int ret; @@ -2810,9 +2227,9 @@ int nand_reset(struct nand_chip *chip, int chipnr) * The CS line has to be released before we can apply the new NAND * interface settings, hence this weird ->select_chip() dance. */ - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, chipnr); ret = nand_reset_op(chip); - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); if (ret) return ret; @@ -2836,6 +2253,48 @@ int nand_reset(struct nand_chip *chip, int chipnr) EXPORT_SYMBOL_GPL(nand_reset); /** + * nand_get_features - wrapper to perform a GET_FEATURE + * @chip: NAND chip info structure + * @addr: feature address + * @subfeature_param: the subfeature parameters, a four bytes array + * + * Returns 0 for success, a negative error otherwise. Returns -ENOTSUPP if the + * operation cannot be handled. + */ +int nand_get_features(struct nand_chip *chip, int addr, + u8 *subfeature_param) +{ + if (!nand_supports_get_features(chip, addr)) + return -ENOTSUPP; + + if (chip->legacy.get_features) + return chip->legacy.get_features(chip, addr, subfeature_param); + + return nand_get_features_op(chip, addr, subfeature_param); +} + +/** + * nand_set_features - wrapper to perform a SET_FEATURE + * @chip: NAND chip info structure + * @addr: feature address + * @subfeature_param: the subfeature parameters, a four bytes array + * + * Returns 0 for success, a negative error otherwise. Returns -ENOTSUPP if the + * operation cannot be handled. + */ +int nand_set_features(struct nand_chip *chip, int addr, + u8 *subfeature_param) +{ + if (!nand_supports_set_features(chip, addr)) + return -ENOTSUPP; + + if (chip->legacy.set_features) + return chip->legacy.set_features(chip, addr, subfeature_param); + + return nand_set_features_op(chip, addr, subfeature_param); +} + +/** * nand_check_erased_buf - check if a buffer contains (almost) only 0xff data * @buf: buffer to test * @len: buffer length @@ -2968,7 +2427,6 @@ EXPORT_SYMBOL(nand_check_erased_ecc_chunk); /** * nand_read_page_raw_notsupp - dummy read raw page function - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: buffer to store read data * @oob_required: caller requires OOB data read to chip->oob_poi @@ -2976,16 +2434,14 @@ EXPORT_SYMBOL(nand_check_erased_ecc_chunk); * * Returns -ENOTSUPP unconditionally. */ -int nand_read_page_raw_notsupp(struct mtd_info *mtd, struct nand_chip *chip, - u8 *buf, int oob_required, int page) +int nand_read_page_raw_notsupp(struct nand_chip *chip, u8 *buf, + int oob_required, int page) { return -ENOTSUPP; } -EXPORT_SYMBOL(nand_read_page_raw_notsupp); /** * nand_read_page_raw - [INTERN] read raw page data without ecc - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: buffer to store read data * @oob_required: caller requires OOB data read to chip->oob_poi @@ -2993,9 +2449,10 @@ EXPORT_SYMBOL(nand_read_page_raw_notsupp); * * Not for syndrome calculating ECC controllers, which use a special oob layout. */ -int nand_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +int nand_read_page_raw(struct nand_chip *chip, uint8_t *buf, int oob_required, + int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int ret; ret = nand_read_page_op(chip, page, 0, buf, mtd->writesize); @@ -3015,7 +2472,6 @@ EXPORT_SYMBOL(nand_read_page_raw); /** * nand_read_page_raw_syndrome - [INTERN] read raw page data without ecc - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: buffer to store read data * @oob_required: caller requires OOB data read to chip->oob_poi @@ -3023,10 +2479,10 @@ EXPORT_SYMBOL(nand_read_page_raw); * * We need a special oob layout and handling even when OOB isn't used. */ -static int nand_read_page_raw_syndrome(struct mtd_info *mtd, - struct nand_chip *chip, uint8_t *buf, +static int nand_read_page_raw_syndrome(struct nand_chip *chip, uint8_t *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int eccsize = chip->ecc.size; int eccbytes = chip->ecc.bytes; uint8_t *oob = chip->oob_poi; @@ -3080,15 +2536,15 @@ static int nand_read_page_raw_syndrome(struct mtd_info *mtd, /** * nand_read_page_swecc - [REPLACEABLE] software ECC based page read function - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: buffer to store read data * @oob_required: caller requires OOB data read to chip->oob_poi * @page: page number to read */ -static int nand_read_page_swecc(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int nand_read_page_swecc(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int i, eccsize = chip->ecc.size, ret; int eccbytes = chip->ecc.bytes; int eccsteps = chip->ecc.steps; @@ -3097,10 +2553,10 @@ static int nand_read_page_swecc(struct mtd_info *mtd, struct nand_chip *chip, uint8_t *ecc_code = chip->ecc.code_buf; unsigned int max_bitflips = 0; - chip->ecc.read_page_raw(mtd, chip, buf, 1, page); + chip->ecc.read_page_raw(chip, buf, 1, page); for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) - chip->ecc.calculate(mtd, p, &ecc_calc[i]); + chip->ecc.calculate(chip, p, &ecc_calc[i]); ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0, chip->ecc.total); @@ -3113,7 +2569,7 @@ static int nand_read_page_swecc(struct mtd_info *mtd, struct nand_chip *chip, for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) { int stat; - stat = chip->ecc.correct(mtd, p, &ecc_code[i], &ecc_calc[i]); + stat = chip->ecc.correct(chip, p, &ecc_code[i], &ecc_calc[i]); if (stat < 0) { mtd->ecc_stats.failed++; } else { @@ -3126,17 +2582,16 @@ static int nand_read_page_swecc(struct mtd_info *mtd, struct nand_chip *chip, /** * nand_read_subpage - [REPLACEABLE] ECC based sub-page read function - * @mtd: mtd info structure * @chip: nand chip info structure * @data_offs: offset of requested data within the page * @readlen: data length * @bufpoi: buffer to store read data * @page: page number to read */ -static int nand_read_subpage(struct mtd_info *mtd, struct nand_chip *chip, - uint32_t data_offs, uint32_t readlen, uint8_t *bufpoi, - int page) +static int nand_read_subpage(struct nand_chip *chip, uint32_t data_offs, + uint32_t readlen, uint8_t *bufpoi, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int start_step, end_step, num_steps, ret; uint8_t *p; int data_col_addr, i, gaps = 0; @@ -3165,7 +2620,7 @@ static int nand_read_subpage(struct mtd_info *mtd, struct nand_chip *chip, /* Calculate ECC */ for (i = 0; i < eccfrag_len ; i += chip->ecc.bytes, p += chip->ecc.size) - chip->ecc.calculate(mtd, p, &chip->ecc.calc_buf[i]); + chip->ecc.calculate(chip, p, &chip->ecc.calc_buf[i]); /* * The performance is faster if we position offsets according to @@ -3214,7 +2669,7 @@ static int nand_read_subpage(struct mtd_info *mtd, struct nand_chip *chip, for (i = 0; i < eccfrag_len ; i += chip->ecc.bytes, p += chip->ecc.size) { int stat; - stat = chip->ecc.correct(mtd, p, &chip->ecc.code_buf[i], + stat = chip->ecc.correct(chip, p, &chip->ecc.code_buf[i], &chip->ecc.calc_buf[i]); if (stat == -EBADMSG && (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) { @@ -3238,7 +2693,6 @@ static int nand_read_subpage(struct mtd_info *mtd, struct nand_chip *chip, /** * nand_read_page_hwecc - [REPLACEABLE] hardware ECC based page read function - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: buffer to store read data * @oob_required: caller requires OOB data read to chip->oob_poi @@ -3246,9 +2700,10 @@ static int nand_read_subpage(struct mtd_info *mtd, struct nand_chip *chip, * * Not for syndrome calculating ECC controllers which need a special oob layout. */ -static int nand_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int nand_read_page_hwecc(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int i, eccsize = chip->ecc.size, ret; int eccbytes = chip->ecc.bytes; int eccsteps = chip->ecc.steps; @@ -3262,13 +2717,13 @@ static int nand_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, return ret; for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) { - chip->ecc.hwctl(mtd, NAND_ECC_READ); + chip->ecc.hwctl(chip, NAND_ECC_READ); ret = nand_read_data_op(chip, p, eccsize, false); if (ret) return ret; - chip->ecc.calculate(mtd, p, &ecc_calc[i]); + chip->ecc.calculate(chip, p, &ecc_calc[i]); } ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false); @@ -3286,7 +2741,7 @@ static int nand_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) { int stat; - stat = chip->ecc.correct(mtd, p, &ecc_code[i], &ecc_calc[i]); + stat = chip->ecc.correct(chip, p, &ecc_code[i], &ecc_calc[i]); if (stat == -EBADMSG && (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) { /* check for empty pages with bitflips */ @@ -3308,7 +2763,6 @@ static int nand_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, /** * nand_read_page_hwecc_oob_first - [REPLACEABLE] hw ecc, read oob first - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: buffer to store read data * @oob_required: caller requires OOB data read to chip->oob_poi @@ -3320,9 +2774,10 @@ static int nand_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, * multiple ECC steps, follows the "infix ECC" scheme and reads/writes ECC from * the data area, by overwriting the NAND manufacturer bad block markings. */ -static int nand_read_page_hwecc_oob_first(struct mtd_info *mtd, - struct nand_chip *chip, uint8_t *buf, int oob_required, int page) +static int nand_read_page_hwecc_oob_first(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int i, eccsize = chip->ecc.size, ret; int eccbytes = chip->ecc.bytes; int eccsteps = chip->ecc.steps; @@ -3348,15 +2803,15 @@ static int nand_read_page_hwecc_oob_first(struct mtd_info *mtd, for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) { int stat; - chip->ecc.hwctl(mtd, NAND_ECC_READ); + chip->ecc.hwctl(chip, NAND_ECC_READ); ret = nand_read_data_op(chip, p, eccsize, false); if (ret) return ret; - chip->ecc.calculate(mtd, p, &ecc_calc[i]); + chip->ecc.calculate(chip, p, &ecc_calc[i]); - stat = chip->ecc.correct(mtd, p, &ecc_code[i], NULL); + stat = chip->ecc.correct(chip, p, &ecc_code[i], NULL); if (stat == -EBADMSG && (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) { /* check for empty pages with bitflips */ @@ -3378,7 +2833,6 @@ static int nand_read_page_hwecc_oob_first(struct mtd_info *mtd, /** * nand_read_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page read - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: buffer to store read data * @oob_required: caller requires OOB data read to chip->oob_poi @@ -3387,9 +2841,10 @@ static int nand_read_page_hwecc_oob_first(struct mtd_info *mtd, * The hw generator calculates the error syndrome automatically. Therefore we * need a special oob layout and handling. */ -static int nand_read_page_syndrome(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int nand_read_page_syndrome(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int ret, i, eccsize = chip->ecc.size; int eccbytes = chip->ecc.bytes; int eccsteps = chip->ecc.steps; @@ -3405,7 +2860,7 @@ static int nand_read_page_syndrome(struct mtd_info *mtd, struct nand_chip *chip, for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) { int stat; - chip->ecc.hwctl(mtd, NAND_ECC_READ); + chip->ecc.hwctl(chip, NAND_ECC_READ); ret = nand_read_data_op(chip, p, eccsize, false); if (ret) @@ -3420,13 +2875,13 @@ static int nand_read_page_syndrome(struct mtd_info *mtd, struct nand_chip *chip, oob += chip->ecc.prepad; } - chip->ecc.hwctl(mtd, NAND_ECC_READSYN); + chip->ecc.hwctl(chip, NAND_ECC_READSYN); ret = nand_read_data_op(chip, oob, eccbytes, false); if (ret) return ret; - stat = chip->ecc.correct(mtd, p, oob, NULL); + stat = chip->ecc.correct(chip, p, oob, NULL); oob += eccbytes; @@ -3502,17 +2957,15 @@ static uint8_t *nand_transfer_oob(struct mtd_info *mtd, uint8_t *oob, /** * nand_setup_read_retry - [INTERN] Set the READ RETRY mode - * @mtd: MTD device structure + * @chip: NAND chip object * @retry_mode: the retry mode to use * * Some vendors supply a special command to shift the Vt threshold, to be used * when there are too many bitflips in a page (i.e., ECC error). After setting * a new threshold, the host should retry reading the page. */ -static int nand_setup_read_retry(struct mtd_info *mtd, int retry_mode) +static int nand_setup_read_retry(struct nand_chip *chip, int retry_mode) { - struct nand_chip *chip = mtd_to_nand(mtd); - pr_debug("setting READ RETRY mode %d\n", retry_mode); if (retry_mode >= chip->read_retries) @@ -3521,7 +2974,18 @@ static int nand_setup_read_retry(struct mtd_info *mtd, int retry_mode) if (!chip->setup_read_retry) return -EOPNOTSUPP; - return chip->setup_read_retry(mtd, retry_mode); + return chip->setup_read_retry(chip, retry_mode); +} + +static void nand_wait_readrdy(struct nand_chip *chip) +{ + const struct nand_sdr_timings *sdr; + + if (!(chip->options & NAND_NEED_READRDY)) + return; + + sdr = nand_get_sdr_timings(&chip->data_interface); + WARN_ON(nand_wait_rdy_op(chip, PSEC_TO_MSEC(sdr->tR_max), 0)); } /** @@ -3549,7 +3013,7 @@ static int nand_do_read_ops(struct mtd_info *mtd, loff_t from, bool ecc_fail = false; chipnr = (int)(from >> chip->chip_shift); - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, chipnr); realpage = (int)(from >> chip->page_shift); page = realpage & chip->pagemask; @@ -3589,16 +3053,15 @@ read_retry: * the read methods return max bitflips per ecc step. */ if (unlikely(ops->mode == MTD_OPS_RAW)) - ret = chip->ecc.read_page_raw(mtd, chip, bufpoi, + ret = chip->ecc.read_page_raw(chip, bufpoi, oob_required, page); else if (!aligned && NAND_HAS_SUBPAGE_READ(chip) && !oob) - ret = chip->ecc.read_subpage(mtd, chip, - col, bytes, bufpoi, - page); + ret = chip->ecc.read_subpage(chip, col, bytes, + bufpoi, page); else - ret = chip->ecc.read_page(mtd, chip, bufpoi, + ret = chip->ecc.read_page(chip, bufpoi, oob_required, page); if (ret < 0) { if (use_bufpoi) @@ -3631,18 +3094,12 @@ read_retry: } } - if (chip->options & NAND_NEED_READRDY) { - /* Apply delay or wait for ready/busy pin */ - if (!chip->dev_ready) - udelay(chip->chip_delay); - else - nand_wait_ready(mtd); - } + nand_wait_readrdy(chip); if (mtd->ecc_stats.failed - ecc_failures) { if (retry_mode + 1 < chip->read_retries) { retry_mode++; - ret = nand_setup_read_retry(mtd, + ret = nand_setup_read_retry(chip, retry_mode); if (ret < 0) break; @@ -3669,7 +3126,7 @@ read_retry: /* Reset to retry mode 0 */ if (retry_mode) { - ret = nand_setup_read_retry(mtd, 0); + ret = nand_setup_read_retry(chip, 0); if (ret < 0) break; retry_mode = 0; @@ -3687,11 +3144,11 @@ read_retry: /* Check, if we cross a chip boundary */ if (!page) { chipnr++; - chip->select_chip(mtd, -1); - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, -1); + chip->select_chip(chip, chipnr); } } - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); ops->retlen = ops->len - (size_t) readlen; if (oob) @@ -3708,12 +3165,13 @@ read_retry: /** * nand_read_oob_std - [REPLACEABLE] the most common OOB data read function - * @mtd: mtd info structure * @chip: nand chip info structure * @page: page number to read */ -int nand_read_oob_std(struct mtd_info *mtd, struct nand_chip *chip, int page) +int nand_read_oob_std(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + return nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize); } EXPORT_SYMBOL(nand_read_oob_std); @@ -3721,13 +3179,12 @@ EXPORT_SYMBOL(nand_read_oob_std); /** * nand_read_oob_syndrome - [REPLACEABLE] OOB data read function for HW ECC * with syndromes - * @mtd: mtd info structure * @chip: nand chip info structure * @page: page number to read */ -int nand_read_oob_syndrome(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int nand_read_oob_syndrome(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int length = mtd->oobsize; int chunk = chip->ecc.bytes + chip->ecc.prepad + chip->ecc.postpad; int eccsize = chip->ecc.size; @@ -3772,16 +3229,16 @@ int nand_read_oob_syndrome(struct mtd_info *mtd, struct nand_chip *chip, return 0; } -EXPORT_SYMBOL(nand_read_oob_syndrome); /** * nand_write_oob_std - [REPLACEABLE] the most common OOB data write function - * @mtd: mtd info structure * @chip: nand chip info structure * @page: page number to write */ -int nand_write_oob_std(struct mtd_info *mtd, struct nand_chip *chip, int page) +int nand_write_oob_std(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + return nand_prog_page_op(chip, page, mtd->writesize, chip->oob_poi, mtd->oobsize); } @@ -3790,13 +3247,12 @@ EXPORT_SYMBOL(nand_write_oob_std); /** * nand_write_oob_syndrome - [REPLACEABLE] OOB data write function for HW ECC * with syndrome - only for large page flash - * @mtd: mtd info structure * @chip: nand chip info structure * @page: page number to write */ -int nand_write_oob_syndrome(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int nand_write_oob_syndrome(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int chunk = chip->ecc.bytes + chip->ecc.prepad + chip->ecc.postpad; int eccsize = chip->ecc.size, length = mtd->oobsize; int ret, i, len, pos, sndcmd = 0, steps = chip->ecc.steps; @@ -3860,7 +3316,6 @@ int nand_write_oob_syndrome(struct mtd_info *mtd, struct nand_chip *chip, return nand_prog_page_end_op(chip); } -EXPORT_SYMBOL(nand_write_oob_syndrome); /** * nand_do_read_oob - [INTERN] NAND read out-of-band @@ -3890,7 +3345,7 @@ static int nand_do_read_oob(struct mtd_info *mtd, loff_t from, len = mtd_oobavail(mtd, ops); chipnr = (int)(from >> chip->chip_shift); - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, chipnr); /* Shift to get page */ realpage = (int)(from >> chip->page_shift); @@ -3898,9 +3353,9 @@ static int nand_do_read_oob(struct mtd_info *mtd, loff_t from, while (1) { if (ops->mode == MTD_OPS_RAW) - ret = chip->ecc.read_oob_raw(mtd, chip, page); + ret = chip->ecc.read_oob_raw(chip, page); else - ret = chip->ecc.read_oob(mtd, chip, page); + ret = chip->ecc.read_oob(chip, page); if (ret < 0) break; @@ -3908,13 +3363,7 @@ static int nand_do_read_oob(struct mtd_info *mtd, loff_t from, len = min(len, readlen); buf = nand_transfer_oob(mtd, buf, ops, len); - if (chip->options & NAND_NEED_READRDY) { - /* Apply delay or wait for ready/busy pin */ - if (!chip->dev_ready) - udelay(chip->chip_delay); - else - nand_wait_ready(mtd); - } + nand_wait_readrdy(chip); max_bitflips = max_t(unsigned int, max_bitflips, ret); @@ -3929,11 +3378,11 @@ static int nand_do_read_oob(struct mtd_info *mtd, loff_t from, /* Check, if we cross a chip boundary */ if (!page) { chipnr++; - chip->select_chip(mtd, -1); - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, -1); + chip->select_chip(chip, chipnr); } } - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); ops->oobretlen = ops->ooblen - readlen; @@ -3979,7 +3428,6 @@ static int nand_read_oob(struct mtd_info *mtd, loff_t from, /** * nand_write_page_raw_notsupp - dummy raw page write function - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: data buffer * @oob_required: must write chip->oob_poi to OOB @@ -3987,16 +3435,14 @@ static int nand_read_oob(struct mtd_info *mtd, loff_t from, * * Returns -ENOTSUPP unconditionally. */ -int nand_write_page_raw_notsupp(struct mtd_info *mtd, struct nand_chip *chip, - const u8 *buf, int oob_required, int page) +int nand_write_page_raw_notsupp(struct nand_chip *chip, const u8 *buf, + int oob_required, int page) { return -ENOTSUPP; } -EXPORT_SYMBOL(nand_write_page_raw_notsupp); /** * nand_write_page_raw - [INTERN] raw page write function - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: data buffer * @oob_required: must write chip->oob_poi to OOB @@ -4004,9 +3450,10 @@ EXPORT_SYMBOL(nand_write_page_raw_notsupp); * * Not for syndrome calculating ECC controllers, which use a special oob layout. */ -int nand_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, int page) +int nand_write_page_raw(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int ret; ret = nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize); @@ -4026,7 +3473,6 @@ EXPORT_SYMBOL(nand_write_page_raw); /** * nand_write_page_raw_syndrome - [INTERN] raw page write function - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: data buffer * @oob_required: must write chip->oob_poi to OOB @@ -4034,11 +3480,11 @@ EXPORT_SYMBOL(nand_write_page_raw); * * We need a special oob layout and handling even when ECC isn't checked. */ -static int nand_write_page_raw_syndrome(struct mtd_info *mtd, - struct nand_chip *chip, +static int nand_write_page_raw_syndrome(struct nand_chip *chip, const uint8_t *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int eccsize = chip->ecc.size; int eccbytes = chip->ecc.bytes; uint8_t *oob = chip->oob_poi; @@ -4091,16 +3537,15 @@ static int nand_write_page_raw_syndrome(struct mtd_info *mtd, } /** * nand_write_page_swecc - [REPLACEABLE] software ECC based page write function - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: data buffer * @oob_required: must write chip->oob_poi to OOB * @page: page number to write */ -static int nand_write_page_swecc(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, - int page) +static int nand_write_page_swecc(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int i, eccsize = chip->ecc.size, ret; int eccbytes = chip->ecc.bytes; int eccsteps = chip->ecc.steps; @@ -4109,28 +3554,27 @@ static int nand_write_page_swecc(struct mtd_info *mtd, struct nand_chip *chip, /* Software ECC calculation */ for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) - chip->ecc.calculate(mtd, p, &ecc_calc[i]); + chip->ecc.calculate(chip, p, &ecc_calc[i]); ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0, chip->ecc.total); if (ret) return ret; - return chip->ecc.write_page_raw(mtd, chip, buf, 1, page); + return chip->ecc.write_page_raw(chip, buf, 1, page); } /** * nand_write_page_hwecc - [REPLACEABLE] hardware ECC based page write function - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: data buffer * @oob_required: must write chip->oob_poi to OOB * @page: page number to write */ -static int nand_write_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, - int page) +static int nand_write_page_hwecc(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int i, eccsize = chip->ecc.size, ret; int eccbytes = chip->ecc.bytes; int eccsteps = chip->ecc.steps; @@ -4142,13 +3586,13 @@ static int nand_write_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, return ret; for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) { - chip->ecc.hwctl(mtd, NAND_ECC_WRITE); + chip->ecc.hwctl(chip, NAND_ECC_WRITE); ret = nand_write_data_op(chip, p, eccsize, false); if (ret) return ret; - chip->ecc.calculate(mtd, p, &ecc_calc[i]); + chip->ecc.calculate(chip, p, &ecc_calc[i]); } ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0, @@ -4166,7 +3610,6 @@ static int nand_write_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, /** * nand_write_subpage_hwecc - [REPLACEABLE] hardware ECC based subpage write - * @mtd: mtd info structure * @chip: nand chip info structure * @offset: column address of subpage within the page * @data_len: data length @@ -4174,11 +3617,11 @@ static int nand_write_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, * @oob_required: must write chip->oob_poi to OOB * @page: page number to write */ -static int nand_write_subpage_hwecc(struct mtd_info *mtd, - struct nand_chip *chip, uint32_t offset, - uint32_t data_len, const uint8_t *buf, - int oob_required, int page) +static int nand_write_subpage_hwecc(struct nand_chip *chip, uint32_t offset, + uint32_t data_len, const uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); uint8_t *oob_buf = chip->oob_poi; uint8_t *ecc_calc = chip->ecc.calc_buf; int ecc_size = chip->ecc.size; @@ -4195,7 +3638,7 @@ static int nand_write_subpage_hwecc(struct mtd_info *mtd, for (step = 0; step < ecc_steps; step++) { /* configure controller for WRITE access */ - chip->ecc.hwctl(mtd, NAND_ECC_WRITE); + chip->ecc.hwctl(chip, NAND_ECC_WRITE); /* write data (untouched subpages already masked by 0xFF) */ ret = nand_write_data_op(chip, buf, ecc_size, false); @@ -4206,7 +3649,7 @@ static int nand_write_subpage_hwecc(struct mtd_info *mtd, if ((step < start_step) || (step > end_step)) memset(ecc_calc, 0xff, ecc_bytes); else - chip->ecc.calculate(mtd, buf, ecc_calc); + chip->ecc.calculate(chip, buf, ecc_calc); /* mask OOB of un-touched subpages by padding 0xFF */ /* if oob_required, preserve OOB metadata of written subpage */ @@ -4237,7 +3680,6 @@ static int nand_write_subpage_hwecc(struct mtd_info *mtd, /** * nand_write_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page write - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: data buffer * @oob_required: must write chip->oob_poi to OOB @@ -4246,11 +3688,10 @@ static int nand_write_subpage_hwecc(struct mtd_info *mtd, * The hw generator calculates the error syndrome automatically. Therefore we * need a special oob layout and handling. */ -static int nand_write_page_syndrome(struct mtd_info *mtd, - struct nand_chip *chip, - const uint8_t *buf, int oob_required, - int page) +static int nand_write_page_syndrome(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int i, eccsize = chip->ecc.size; int eccbytes = chip->ecc.bytes; int eccsteps = chip->ecc.steps; @@ -4263,7 +3704,7 @@ static int nand_write_page_syndrome(struct mtd_info *mtd, return ret; for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) { - chip->ecc.hwctl(mtd, NAND_ECC_WRITE); + chip->ecc.hwctl(chip, NAND_ECC_WRITE); ret = nand_write_data_op(chip, p, eccsize, false); if (ret) @@ -4278,7 +3719,7 @@ static int nand_write_page_syndrome(struct mtd_info *mtd, oob += chip->ecc.prepad; } - chip->ecc.calculate(mtd, p, oob); + chip->ecc.calculate(chip, p, oob); ret = nand_write_data_op(chip, oob, eccbytes, false); if (ret) @@ -4331,14 +3772,13 @@ static int nand_write_page(struct mtd_info *mtd, struct nand_chip *chip, subpage = 0; if (unlikely(raw)) - status = chip->ecc.write_page_raw(mtd, chip, buf, - oob_required, page); + status = chip->ecc.write_page_raw(chip, buf, oob_required, + page); else if (subpage) - status = chip->ecc.write_subpage(mtd, chip, offset, data_len, - buf, oob_required, page); + status = chip->ecc.write_subpage(chip, offset, data_len, buf, + oob_required, page); else - status = chip->ecc.write_page(mtd, chip, buf, oob_required, - page); + status = chip->ecc.write_page(chip, buf, oob_required, page); if (status < 0) return status; @@ -4423,7 +3863,7 @@ static int nand_do_write_ops(struct mtd_info *mtd, loff_t to, column = to & (mtd->writesize - 1); chipnr = (int)(to >> chip->chip_shift); - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, chipnr); /* Check, if it is write protected */ if (nand_check_wp(mtd)) { @@ -4499,8 +3939,8 @@ static int nand_do_write_ops(struct mtd_info *mtd, loff_t to, /* Check, if we cross a chip boundary */ if (!page) { chipnr++; - chip->select_chip(mtd, -1); - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, -1); + chip->select_chip(chip, chipnr); } } @@ -4509,7 +3949,7 @@ static int nand_do_write_ops(struct mtd_info *mtd, loff_t to, ops->oobretlen = ops->ooblen; err_out: - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); return ret; } @@ -4535,10 +3975,10 @@ static int panic_nand_write(struct mtd_info *mtd, loff_t to, size_t len, /* Grab the device */ panic_nand_get_device(chip, mtd, FL_WRITING); - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, chipnr); /* Wait for the device to get ready */ - panic_nand_wait(mtd, chip, 400); + panic_nand_wait(chip, 400); memset(&ops, 0, sizeof(ops)); ops.len = len; @@ -4587,14 +4027,14 @@ static int nand_do_write_oob(struct mtd_info *mtd, loff_t to, */ nand_reset(chip, chipnr); - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, chipnr); /* Shift to get page */ page = (int)(to >> chip->page_shift); /* Check, if it is write protected */ if (nand_check_wp(mtd)) { - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); return -EROFS; } @@ -4605,11 +4045,11 @@ static int nand_do_write_oob(struct mtd_info *mtd, loff_t to, nand_fill_oob(mtd, ops->oobbuf, ops->ooblen, ops); if (ops->mode == MTD_OPS_RAW) - status = chip->ecc.write_oob_raw(mtd, chip, page & chip->pagemask); + status = chip->ecc.write_oob_raw(chip, page & chip->pagemask); else - status = chip->ecc.write_oob(mtd, chip, page & chip->pagemask); + status = chip->ecc.write_oob(chip, page & chip->pagemask); - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); if (status) return status; @@ -4656,14 +4096,13 @@ out: /** * single_erase - [GENERIC] NAND standard block erase command function - * @mtd: MTD device structure + * @chip: NAND chip object * @page: the page address of the block which will be erased * * Standard erase command for NAND chips. Returns NAND status. */ -static int single_erase(struct mtd_info *mtd, int page) +static int single_erase(struct nand_chip *chip, int page) { - struct nand_chip *chip = mtd_to_nand(mtd); unsigned int eraseblock; /* Send commands to erase a block */ @@ -4681,22 +4120,22 @@ static int single_erase(struct mtd_info *mtd, int page) */ static int nand_erase(struct mtd_info *mtd, struct erase_info *instr) { - return nand_erase_nand(mtd, instr, 0); + return nand_erase_nand(mtd_to_nand(mtd), instr, 0); } /** * nand_erase_nand - [INTERN] erase block(s) - * @mtd: MTD device structure + * @chip: NAND chip object * @instr: erase instruction * @allowbbt: allow erasing the bbt area * * Erase one ore more blocks. */ -int nand_erase_nand(struct mtd_info *mtd, struct erase_info *instr, +int nand_erase_nand(struct nand_chip *chip, struct erase_info *instr, int allowbbt) { + struct mtd_info *mtd = nand_to_mtd(chip); int page, status, pages_per_block, ret, chipnr; - struct nand_chip *chip = mtd_to_nand(mtd); loff_t len; pr_debug("%s: start = 0x%012llx, len = %llu\n", @@ -4717,7 +4156,7 @@ int nand_erase_nand(struct mtd_info *mtd, struct erase_info *instr, pages_per_block = 1 << (chip->phys_erase_shift - chip->page_shift); /* Select the NAND device */ - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, chipnr); /* Check, if it is write protected */ if (nand_check_wp(mtd)) { @@ -4748,7 +4187,11 @@ int nand_erase_nand(struct mtd_info *mtd, struct erase_info *instr, (page + pages_per_block)) chip->pagebuf = -1; - status = chip->erase(mtd, page & chip->pagemask); + if (chip->legacy.erase) + status = chip->legacy.erase(chip, + page & chip->pagemask); + else + status = single_erase(chip, page & chip->pagemask); /* See if block erase succeeded */ if (status) { @@ -4767,8 +4210,8 @@ int nand_erase_nand(struct mtd_info *mtd, struct erase_info *instr, /* Check, if we cross a chip boundary */ if (len && !(page & chip->pagemask)) { chipnr++; - chip->select_chip(mtd, -1); - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, -1); + chip->select_chip(chip, chipnr); } } @@ -4776,7 +4219,7 @@ int nand_erase_nand(struct mtd_info *mtd, struct erase_info *instr, erase_exit: /* Deselect and wake up anyone waiting on the device */ - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); nand_release_device(mtd); /* Return more or less happy */ @@ -4812,11 +4255,11 @@ static int nand_block_isbad(struct mtd_info *mtd, loff_t offs) /* Select the NAND device */ nand_get_device(mtd, FL_READING); - chip->select_chip(mtd, chipnr); + chip->select_chip(chip, chipnr); ret = nand_block_checkbad(mtd, offs, 0); - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); nand_release_device(mtd); return ret; @@ -4879,51 +4322,6 @@ static int nand_max_bad_blocks(struct mtd_info *mtd, loff_t ofs, size_t len) } /** - * nand_default_set_features- [REPLACEABLE] set NAND chip features - * @mtd: MTD device structure - * @chip: nand chip info structure - * @addr: feature address. - * @subfeature_param: the subfeature parameters, a four bytes array. - */ -static int nand_default_set_features(struct mtd_info *mtd, - struct nand_chip *chip, int addr, - uint8_t *subfeature_param) -{ - return nand_set_features_op(chip, addr, subfeature_param); -} - -/** - * nand_default_get_features- [REPLACEABLE] get NAND chip features - * @mtd: MTD device structure - * @chip: nand chip info structure - * @addr: feature address. - * @subfeature_param: the subfeature parameters, a four bytes array. - */ -static int nand_default_get_features(struct mtd_info *mtd, - struct nand_chip *chip, int addr, - uint8_t *subfeature_param) -{ - return nand_get_features_op(chip, addr, subfeature_param); -} - -/** - * nand_get_set_features_notsupp - set/get features stub returning -ENOTSUPP - * @mtd: MTD device structure - * @chip: nand chip info structure - * @addr: feature address. - * @subfeature_param: the subfeature parameters, a four bytes array. - * - * Should be used by NAND controller drivers that do not support the SET/GET - * FEATURES operations. - */ -int nand_get_set_features_notsupp(struct mtd_info *mtd, struct nand_chip *chip, - int addr, u8 *subfeature_param) -{ - return -ENOTSUPP; -} -EXPORT_SYMBOL(nand_get_set_features_notsupp); - -/** * nand_suspend - [MTD Interface] Suspend the NAND flash * @mtd: MTD device structure */ @@ -4960,44 +4358,7 @@ static void nand_shutdown(struct mtd_info *mtd) /* Set default functions */ static void nand_set_defaults(struct nand_chip *chip) { - unsigned int busw = chip->options & NAND_BUSWIDTH_16; - - /* check for proper chip_delay setup, set 20us if not */ - if (!chip->chip_delay) - chip->chip_delay = 20; - - /* check, if a user supplied command function given */ - if (!chip->cmdfunc && !chip->exec_op) - chip->cmdfunc = nand_command; - - /* check, if a user supplied wait function given */ - if (chip->waitfunc == NULL) - chip->waitfunc = nand_wait; - - if (!chip->select_chip) - chip->select_chip = nand_select_chip; - - /* set for ONFI nand */ - if (!chip->set_features) - chip->set_features = nand_default_set_features; - if (!chip->get_features) - chip->get_features = nand_default_get_features; - - /* If called twice, pointers that depend on busw may need to be reset */ - if (!chip->read_byte || chip->read_byte == nand_read_byte) - chip->read_byte = busw ? nand_read_byte16 : nand_read_byte; - if (!chip->read_word) - chip->read_word = nand_read_word; - if (!chip->block_bad) - chip->block_bad = nand_block_bad; - if (!chip->block_markbad) - chip->block_markbad = nand_default_block_markbad; - if (!chip->write_buf || chip->write_buf == nand_write_buf) - chip->write_buf = busw ? nand_write_buf16 : nand_write_buf; - if (!chip->write_byte || chip->write_byte == nand_write_byte) - chip->write_byte = busw ? nand_write_byte16 : nand_write_byte; - if (!chip->read_buf || chip->read_buf == nand_read_buf) - chip->read_buf = busw ? nand_read_buf16 : nand_read_buf; + nand_legacy_set_defaults(chip); if (!chip->controller) { chip->controller = &chip->dummy_controller; @@ -5009,7 +4370,7 @@ static void nand_set_defaults(struct nand_chip *chip) } /* Sanitize ONFI strings so we can safely print them */ -static void sanitize_string(uint8_t *s, size_t len) +void sanitize_string(uint8_t *s, size_t len) { ssize_t i; @@ -5026,390 +4387,6 @@ static void sanitize_string(uint8_t *s, size_t len) strim(s); } -static u16 onfi_crc16(u16 crc, u8 const *p, size_t len) -{ - int i; - while (len--) { - crc ^= *p++ << 8; - for (i = 0; i < 8; i++) - crc = (crc << 1) ^ ((crc & 0x8000) ? 0x8005 : 0); - } - - return crc; -} - -/* Parse the Extended Parameter Page. */ -static int nand_flash_detect_ext_param_page(struct nand_chip *chip, - struct nand_onfi_params *p) -{ - struct onfi_ext_param_page *ep; - struct onfi_ext_section *s; - struct onfi_ext_ecc_info *ecc; - uint8_t *cursor; - int ret; - int len; - int i; - - len = le16_to_cpu(p->ext_param_page_length) * 16; - ep = kmalloc(len, GFP_KERNEL); - if (!ep) - return -ENOMEM; - - /* Send our own NAND_CMD_PARAM. */ - ret = nand_read_param_page_op(chip, 0, NULL, 0); - if (ret) - goto ext_out; - - /* Use the Change Read Column command to skip the ONFI param pages. */ - ret = nand_change_read_column_op(chip, - sizeof(*p) * p->num_of_param_pages, - ep, len, true); - if (ret) - goto ext_out; - - ret = -EINVAL; - if ((onfi_crc16(ONFI_CRC_BASE, ((uint8_t *)ep) + 2, len - 2) - != le16_to_cpu(ep->crc))) { - pr_debug("fail in the CRC.\n"); - goto ext_out; - } - - /* - * Check the signature. - * Do not strictly follow the ONFI spec, maybe changed in future. - */ - if (strncmp(ep->sig, "EPPS", 4)) { - pr_debug("The signature is invalid.\n"); - goto ext_out; - } - - /* find the ECC section. */ - cursor = (uint8_t *)(ep + 1); - for (i = 0; i < ONFI_EXT_SECTION_MAX; i++) { - s = ep->sections + i; - if (s->type == ONFI_SECTION_TYPE_2) - break; - cursor += s->length * 16; - } - if (i == ONFI_EXT_SECTION_MAX) { - pr_debug("We can not find the ECC section.\n"); - goto ext_out; - } - - /* get the info we want. */ - ecc = (struct onfi_ext_ecc_info *)cursor; - - if (!ecc->codeword_size) { - pr_debug("Invalid codeword size\n"); - goto ext_out; - } - - chip->ecc_strength_ds = ecc->ecc_bits; - chip->ecc_step_ds = 1 << ecc->codeword_size; - ret = 0; - -ext_out: - kfree(ep); - return ret; -} - -/* - * Recover data with bit-wise majority - */ -static void nand_bit_wise_majority(const void **srcbufs, - unsigned int nsrcbufs, - void *dstbuf, - unsigned int bufsize) -{ - int i, j, k; - - for (i = 0; i < bufsize; i++) { - u8 val = 0; - - for (j = 0; j < 8; j++) { - unsigned int cnt = 0; - - for (k = 0; k < nsrcbufs; k++) { - const u8 *srcbuf = srcbufs[k]; - - if (srcbuf[i] & BIT(j)) - cnt++; - } - - if (cnt > nsrcbufs / 2) - val |= BIT(j); - } - - ((u8 *)dstbuf)[i] = val; - } -} - -/* - * Check if the NAND chip is ONFI compliant, returns 1 if it is, 0 otherwise. - */ -static int nand_flash_detect_onfi(struct nand_chip *chip) -{ - struct mtd_info *mtd = nand_to_mtd(chip); - struct nand_onfi_params *p; - struct onfi_params *onfi; - int onfi_version = 0; - char id[4]; - int i, ret, val; - - /* Try ONFI for unknown chip or LP */ - ret = nand_readid_op(chip, 0x20, id, sizeof(id)); - if (ret || strncmp(id, "ONFI", 4)) - return 0; - - /* ONFI chip: allocate a buffer to hold its parameter page */ - p = kzalloc((sizeof(*p) * 3), GFP_KERNEL); - if (!p) - return -ENOMEM; - - ret = nand_read_param_page_op(chip, 0, NULL, 0); - if (ret) { - ret = 0; - goto free_onfi_param_page; - } - - for (i = 0; i < 3; i++) { - ret = nand_read_data_op(chip, &p[i], sizeof(*p), true); - if (ret) { - ret = 0; - goto free_onfi_param_page; - } - - if (onfi_crc16(ONFI_CRC_BASE, (u8 *)&p[i], 254) == - le16_to_cpu(p->crc)) { - if (i) - memcpy(p, &p[i], sizeof(*p)); - break; - } - } - - if (i == 3) { - const void *srcbufs[3] = {p, p + 1, p + 2}; - - pr_warn("Could not find a valid ONFI parameter page, trying bit-wise majority to recover it\n"); - nand_bit_wise_majority(srcbufs, ARRAY_SIZE(srcbufs), p, - sizeof(*p)); - - if (onfi_crc16(ONFI_CRC_BASE, (u8 *)p, 254) != - le16_to_cpu(p->crc)) { - pr_err("ONFI parameter recovery failed, aborting\n"); - goto free_onfi_param_page; - } - } - - if (chip->manufacturer.desc && chip->manufacturer.desc->ops && - chip->manufacturer.desc->ops->fixup_onfi_param_page) - chip->manufacturer.desc->ops->fixup_onfi_param_page(chip, p); - - /* Check version */ - val = le16_to_cpu(p->revision); - if (val & ONFI_VERSION_2_3) - onfi_version = 23; - else if (val & ONFI_VERSION_2_2) - onfi_version = 22; - else if (val & ONFI_VERSION_2_1) - onfi_version = 21; - else if (val & ONFI_VERSION_2_0) - onfi_version = 20; - else if (val & ONFI_VERSION_1_0) - onfi_version = 10; - - if (!onfi_version) { - pr_info("unsupported ONFI version: %d\n", val); - goto free_onfi_param_page; - } - - sanitize_string(p->manufacturer, sizeof(p->manufacturer)); - sanitize_string(p->model, sizeof(p->model)); - chip->parameters.model = kstrdup(p->model, GFP_KERNEL); - if (!chip->parameters.model) { - ret = -ENOMEM; - goto free_onfi_param_page; - } - - mtd->writesize = le32_to_cpu(p->byte_per_page); - - /* - * pages_per_block and blocks_per_lun may not be a power-of-2 size - * (don't ask me who thought of this...). MTD assumes that these - * dimensions will be power-of-2, so just truncate the remaining area. - */ - mtd->erasesize = 1 << (fls(le32_to_cpu(p->pages_per_block)) - 1); - mtd->erasesize *= mtd->writesize; - - mtd->oobsize = le16_to_cpu(p->spare_bytes_per_page); - - /* See erasesize comment */ - chip->chipsize = 1 << (fls(le32_to_cpu(p->blocks_per_lun)) - 1); - chip->chipsize *= (uint64_t)mtd->erasesize * p->lun_count; - chip->bits_per_cell = p->bits_per_cell; - - chip->max_bb_per_die = le16_to_cpu(p->bb_per_lun); - chip->blocks_per_die = le32_to_cpu(p->blocks_per_lun); - - if (le16_to_cpu(p->features) & ONFI_FEATURE_16_BIT_BUS) - chip->options |= NAND_BUSWIDTH_16; - - if (p->ecc_bits != 0xff) { - chip->ecc_strength_ds = p->ecc_bits; - chip->ecc_step_ds = 512; - } else if (onfi_version >= 21 && - (le16_to_cpu(p->features) & ONFI_FEATURE_EXT_PARAM_PAGE)) { - - /* - * The nand_flash_detect_ext_param_page() uses the - * Change Read Column command which maybe not supported - * by the chip->cmdfunc. So try to update the chip->cmdfunc - * now. We do not replace user supplied command function. - */ - if (mtd->writesize > 512 && chip->cmdfunc == nand_command) - chip->cmdfunc = nand_command_lp; - - /* The Extended Parameter Page is supported since ONFI 2.1. */ - if (nand_flash_detect_ext_param_page(chip, p)) - pr_warn("Failed to detect ONFI extended param page\n"); - } else { - pr_warn("Could not retrieve ONFI ECC requirements\n"); - } - - /* Save some parameters from the parameter page for future use */ - if (le16_to_cpu(p->opt_cmd) & ONFI_OPT_CMD_SET_GET_FEATURES) { - chip->parameters.supports_set_get_features = true; - bitmap_set(chip->parameters.get_feature_list, - ONFI_FEATURE_ADDR_TIMING_MODE, 1); - bitmap_set(chip->parameters.set_feature_list, - ONFI_FEATURE_ADDR_TIMING_MODE, 1); - } - - onfi = kzalloc(sizeof(*onfi), GFP_KERNEL); - if (!onfi) { - ret = -ENOMEM; - goto free_model; - } - - onfi->version = onfi_version; - onfi->tPROG = le16_to_cpu(p->t_prog); - onfi->tBERS = le16_to_cpu(p->t_bers); - onfi->tR = le16_to_cpu(p->t_r); - onfi->tCCS = le16_to_cpu(p->t_ccs); - onfi->async_timing_mode = le16_to_cpu(p->async_timing_mode); - onfi->vendor_revision = le16_to_cpu(p->vendor_revision); - memcpy(onfi->vendor, p->vendor, sizeof(p->vendor)); - chip->parameters.onfi = onfi; - - /* Identification done, free the full ONFI parameter page and exit */ - kfree(p); - - return 1; - -free_model: - kfree(chip->parameters.model); -free_onfi_param_page: - kfree(p); - - return ret; -} - -/* - * Check if the NAND chip is JEDEC compliant, returns 1 if it is, 0 otherwise. - */ -static int nand_flash_detect_jedec(struct nand_chip *chip) -{ - struct mtd_info *mtd = nand_to_mtd(chip); - struct nand_jedec_params *p; - struct jedec_ecc_info *ecc; - int jedec_version = 0; - char id[5]; - int i, val, ret; - - /* Try JEDEC for unknown chip or LP */ - ret = nand_readid_op(chip, 0x40, id, sizeof(id)); - if (ret || strncmp(id, "JEDEC", sizeof(id))) - return 0; - - /* JEDEC chip: allocate a buffer to hold its parameter page */ - p = kzalloc(sizeof(*p), GFP_KERNEL); - if (!p) - return -ENOMEM; - - ret = nand_read_param_page_op(chip, 0x40, NULL, 0); - if (ret) { - ret = 0; - goto free_jedec_param_page; - } - - for (i = 0; i < 3; i++) { - ret = nand_read_data_op(chip, p, sizeof(*p), true); - if (ret) { - ret = 0; - goto free_jedec_param_page; - } - - if (onfi_crc16(ONFI_CRC_BASE, (uint8_t *)p, 510) == - le16_to_cpu(p->crc)) - break; - } - - if (i == 3) { - pr_err("Could not find valid JEDEC parameter page; aborting\n"); - goto free_jedec_param_page; - } - - /* Check version */ - val = le16_to_cpu(p->revision); - if (val & (1 << 2)) - jedec_version = 10; - else if (val & (1 << 1)) - jedec_version = 1; /* vendor specific version */ - - if (!jedec_version) { - pr_info("unsupported JEDEC version: %d\n", val); - goto free_jedec_param_page; - } - - sanitize_string(p->manufacturer, sizeof(p->manufacturer)); - sanitize_string(p->model, sizeof(p->model)); - chip->parameters.model = kstrdup(p->model, GFP_KERNEL); - if (!chip->parameters.model) { - ret = -ENOMEM; - goto free_jedec_param_page; - } - - mtd->writesize = le32_to_cpu(p->byte_per_page); - - /* Please reference to the comment for nand_flash_detect_onfi. */ - mtd->erasesize = 1 << (fls(le32_to_cpu(p->pages_per_block)) - 1); - mtd->erasesize *= mtd->writesize; - - mtd->oobsize = le16_to_cpu(p->spare_bytes_per_page); - - /* Please reference to the comment for nand_flash_detect_onfi. */ - chip->chipsize = 1 << (fls(le32_to_cpu(p->blocks_per_lun)) - 1); - chip->chipsize *= (uint64_t)mtd->erasesize * p->lun_count; - chip->bits_per_cell = p->bits_per_cell; - - if (le16_to_cpu(p->features) & JEDEC_FEATURE_16_BIT_BUS) - chip->options |= NAND_BUSWIDTH_16; - - /* ECC info */ - ecc = &p->ecc_info[0]; - - if (ecc->codeword_size >= 9) { - chip->ecc_strength_ds = ecc->ecc_bits; - chip->ecc_step_ds = 1 << ecc->codeword_size; - } else { - pr_warn("Invalid codeword size\n"); - } - -free_jedec_param_page: - kfree(p); - return ret; -} - /* * nand_id_has_period - Check if an ID string has a given wraparound period * @id_data: the ID string @@ -5625,6 +4602,12 @@ static void nand_manufacturer_cleanup(struct nand_chip *chip) chip->manufacturer.desc->ops->cleanup(chip); } +static const char * +nand_manufacturer_name(const struct nand_manufacturer *manufacturer) +{ + return manufacturer ? manufacturer->name : "Unknown"; +} + /* * Get the flash and manufacturer id and lookup if the type is supported. */ @@ -5645,7 +4628,7 @@ static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type) return ret; /* Select the device */ - chip->select_chip(mtd, 0); + chip->select_chip(chip, 0); /* Send the command for reading device ID */ ret = nand_readid_op(chip, 0, id_data, 2); @@ -5709,14 +4692,14 @@ static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type) if (!type->name || !type->pagesize) { /* Check if the chip is ONFI compliant */ - ret = nand_flash_detect_onfi(chip); + ret = nand_onfi_detect(chip); if (ret < 0) return ret; else if (ret) goto ident_done; /* Check if the chip is JEDEC compliant */ - ret = nand_flash_detect_jedec(chip); + ret = nand_jedec_detect(chip); if (ret < 0) return ret; else if (ret) @@ -5783,11 +4766,8 @@ ident_done: chip->options |= NAND_ROW_ADDR_3; chip->badblockbits = 8; - chip->erase = single_erase; - /* Do not replace user supplied command function! */ - if (mtd->writesize > 512 && chip->cmdfunc == nand_command) - chip->cmdfunc = nand_command_lp; + nand_legacy_adjust_cmdfunc(chip); pr_info("device found, Manufacturer ID: 0x%02x, Chip ID: 0x%02x\n", maf_id, dev_id); @@ -5953,7 +4933,7 @@ static int nand_dt_init(struct nand_chip *chip) /** * nand_scan_ident - Scan for the NAND device - * @mtd: MTD device structure + * @chip: NAND chip object * @maxchips: number of chips to scan for * @table: alternative NAND ID table * @@ -5965,11 +4945,12 @@ static int nand_dt_init(struct nand_chip *chip) * prevented dynamic allocations during this phase which was unconvenient and * as been banned for the benefit of the ->init_ecc()/cleanup_ecc() hooks. */ -static int nand_scan_ident(struct mtd_info *mtd, int maxchips, +static int nand_scan_ident(struct nand_chip *chip, unsigned int maxchips, struct nand_flash_dev *table) { - int i, nand_maf_id, nand_dev_id; - struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(chip); + int nand_maf_id, nand_dev_id; + unsigned int i; int ret; /* Enforce the right timings for reset/detection */ @@ -5982,21 +4963,15 @@ static int nand_scan_ident(struct mtd_info *mtd, int maxchips, if (!mtd->name && mtd->dev.parent) mtd->name = dev_name(mtd->dev.parent); - /* - * ->cmdfunc() is legacy and will only be used if ->exec_op() is not - * populated. - */ - if (!chip->exec_op) { - /* - * Default functions assigned for ->cmdfunc() and - * ->select_chip() both expect ->cmd_ctrl() to be populated. - */ - if ((!chip->cmdfunc || !chip->select_chip) && !chip->cmd_ctrl) { - pr_err("->cmd_ctrl() should be provided\n"); - return -EINVAL; - } + if (chip->exec_op && !chip->select_chip) { + pr_err("->select_chip() is mandatory when implementing ->exec_op()\n"); + return -EINVAL; } + ret = nand_legacy_check_hooks(chip); + if (ret) + return ret; + /* Set the default functions */ nand_set_defaults(chip); @@ -6005,14 +4980,14 @@ static int nand_scan_ident(struct mtd_info *mtd, int maxchips, if (ret) { if (!(chip->options & NAND_SCAN_SILENT_NODEV)) pr_warn("No NAND device found\n"); - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); return ret; } nand_maf_id = chip->id.data[0]; nand_dev_id = chip->id.data[1]; - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); /* Check for a chip array */ for (i = 1; i < maxchips; i++) { @@ -6021,15 +4996,15 @@ static int nand_scan_ident(struct mtd_info *mtd, int maxchips, /* See comment in nand_get_flash_type for reset */ nand_reset(chip, i); - chip->select_chip(mtd, i); + chip->select_chip(chip, i); /* Send the command for reading device ID */ nand_readid_op(chip, 0, id, sizeof(id)); /* Read manufacturer and device IDs */ if (nand_maf_id != id[0] || nand_dev_id != id[1]) { - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); break; } - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); } if (i > 1) pr_info("%d chips detected\n", i); @@ -6070,6 +5045,10 @@ static int nand_set_ecc_soft_ops(struct mtd_info *mtd) ecc->size = 256; ecc->bytes = 3; ecc->strength = 1; + + if (IS_ENABLED(CONFIG_MTD_NAND_ECC_SMC)) + ecc->options |= NAND_ECC_SOFT_HAMMING_SM_ORDER; + return 0; case NAND_ECC_BCH: if (!mtd_nand_has_bch()) { @@ -6423,15 +5402,15 @@ static bool nand_ecc_strength_good(struct mtd_info *mtd) /** * nand_scan_tail - Scan for the NAND device - * @mtd: MTD device structure + * @chip: NAND chip object * * This is the second phase of the normal nand_scan() function. It fills out * all the uninitialized function pointers with the defaults and scans for a * bad block table if appropriate. */ -static int nand_scan_tail(struct mtd_info *mtd) +static int nand_scan_tail(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(chip); struct nand_ecc_ctrl *ecc = &chip->ecc; int ret, i; @@ -6451,9 +5430,9 @@ static int nand_scan_tail(struct mtd_info *mtd) * to explictly select the relevant die when interacting with the NAND * chip. */ - chip->select_chip(mtd, 0); + chip->select_chip(chip, 0); ret = nand_manufacturer_init(chip); - chip->select_chip(mtd, -1); + chip->select_chip(chip, -1); if (ret) goto err_free_buf; @@ -6770,33 +5749,31 @@ static void nand_detach(struct nand_chip *chip) /** * nand_scan_with_ids - [NAND Interface] Scan for the NAND device - * @mtd: MTD device structure - * @maxchips: number of chips to scan for. @nand_scan_ident() will not be run if - * this parameter is zero (useful for specific drivers that must - * handle this part of the process themselves, e.g docg4). + * @chip: NAND chip object + * @maxchips: number of chips to scan for. * @ids: optional flash IDs table * * This fills out all the uninitialized function pointers with the defaults. * The flash ID is read and the mtd/chip structures are filled with the * appropriate values. */ -int nand_scan_with_ids(struct mtd_info *mtd, int maxchips, +int nand_scan_with_ids(struct nand_chip *chip, unsigned int maxchips, struct nand_flash_dev *ids) { - struct nand_chip *chip = mtd_to_nand(mtd); int ret; - if (maxchips) { - ret = nand_scan_ident(mtd, maxchips, ids); - if (ret) - return ret; - } + if (!maxchips) + return -EINVAL; + + ret = nand_scan_ident(chip, maxchips, ids); + if (ret) + return ret; ret = nand_attach(chip); if (ret) goto cleanup_ident; - ret = nand_scan_tail(mtd); + ret = nand_scan_tail(chip); if (ret) goto detach_chip; @@ -6847,12 +5824,12 @@ EXPORT_SYMBOL_GPL(nand_cleanup); /** * nand_release - [NAND Interface] Unregister the MTD device and free resources * held by the NAND device - * @mtd: MTD device structure + * @chip: NAND chip object */ -void nand_release(struct mtd_info *mtd) +void nand_release(struct nand_chip *chip) { - mtd_device_unregister(mtd); - nand_cleanup(mtd_to_nand(mtd)); + mtd_device_unregister(nand_to_mtd(chip)); + nand_cleanup(chip); } EXPORT_SYMBOL_GPL(nand_release); diff --git a/drivers/mtd/nand/raw/nand_bbt.c b/drivers/mtd/nand/raw/nand_bbt.c index 39db352f8757..98a826838b60 100644 --- a/drivers/mtd/nand/raw/nand_bbt.c +++ b/drivers/mtd/nand/raw/nand_bbt.c @@ -61,13 +61,14 @@ #include <linux/types.h> #include <linux/mtd/mtd.h> #include <linux/mtd/bbm.h> -#include <linux/mtd/rawnand.h> #include <linux/bitops.h> #include <linux/delay.h> #include <linux/vmalloc.h> #include <linux/export.h> #include <linux/string.h> +#include "internals.h" + #define BBT_BLOCK_GOOD 0x00 #define BBT_BLOCK_WORN 0x01 #define BBT_BLOCK_RESERVED 0x02 @@ -683,14 +684,13 @@ static void mark_bbt_block_bad(struct nand_chip *this, struct nand_bbt_descr *td, int chip, int block) { - struct mtd_info *mtd = nand_to_mtd(this); loff_t to; int res; bbt_mark_entry(this, block, BBT_BLOCK_WORN); to = (loff_t)block << this->bbt_erase_shift; - res = this->block_markbad(mtd, to); + res = nand_markbad_bbm(this, to); if (res) pr_warn("nand_bbt: error %d while marking block %d bad\n", res, block); @@ -854,7 +854,7 @@ static int write_bbt(struct mtd_info *mtd, uint8_t *buf, memset(&einfo, 0, sizeof(einfo)); einfo.addr = to; einfo.len = 1 << this->bbt_erase_shift; - res = nand_erase_nand(mtd, &einfo, 1); + res = nand_erase_nand(this, &einfo, 1); if (res < 0) { pr_warn("nand_bbt: error while erasing BBT block %d\n", res); @@ -1388,12 +1388,11 @@ EXPORT_SYMBOL(nand_create_bbt); /** * nand_isreserved_bbt - [NAND Interface] Check if a block is reserved - * @mtd: MTD device structure + * @this: NAND chip object * @offs: offset in the device */ -int nand_isreserved_bbt(struct mtd_info *mtd, loff_t offs) +int nand_isreserved_bbt(struct nand_chip *this, loff_t offs) { - struct nand_chip *this = mtd_to_nand(mtd); int block; block = (int)(offs >> this->bbt_erase_shift); @@ -1402,13 +1401,12 @@ int nand_isreserved_bbt(struct mtd_info *mtd, loff_t offs) /** * nand_isbad_bbt - [NAND Interface] Check if a block is bad - * @mtd: MTD device structure + * @this: NAND chip object * @offs: offset in the device * @allowbbt: allow access to bad block table region */ -int nand_isbad_bbt(struct mtd_info *mtd, loff_t offs, int allowbbt) +int nand_isbad_bbt(struct nand_chip *this, loff_t offs, int allowbbt) { - struct nand_chip *this = mtd_to_nand(mtd); int block, res; block = (int)(offs >> this->bbt_erase_shift); @@ -1430,12 +1428,12 @@ int nand_isbad_bbt(struct mtd_info *mtd, loff_t offs, int allowbbt) /** * nand_markbad_bbt - [NAND Interface] Mark a block bad in the BBT - * @mtd: MTD device structure + * @this: NAND chip object * @offs: offset of the bad block */ -int nand_markbad_bbt(struct mtd_info *mtd, loff_t offs) +int nand_markbad_bbt(struct nand_chip *this, loff_t offs) { - struct nand_chip *this = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(this); int block, ret = 0; block = (int)(offs >> this->bbt_erase_shift); diff --git a/drivers/mtd/nand/raw/nand_bch.c b/drivers/mtd/nand/raw/nand_bch.c index b7387ace567a..574c0ca16160 100644 --- a/drivers/mtd/nand/raw/nand_bch.c +++ b/drivers/mtd/nand/raw/nand_bch.c @@ -43,14 +43,13 @@ struct nand_bch_control { /** * nand_bch_calculate_ecc - [NAND Interface] Calculate ECC for data block - * @mtd: MTD block structure + * @chip: NAND chip object * @buf: input buffer with raw data * @code: output buffer with ECC */ -int nand_bch_calculate_ecc(struct mtd_info *mtd, const unsigned char *buf, +int nand_bch_calculate_ecc(struct nand_chip *chip, const unsigned char *buf, unsigned char *code) { - const struct nand_chip *chip = mtd_to_nand(mtd); struct nand_bch_control *nbc = chip->ecc.priv; unsigned int i; @@ -67,17 +66,16 @@ EXPORT_SYMBOL(nand_bch_calculate_ecc); /** * nand_bch_correct_data - [NAND Interface] Detect and correct bit error(s) - * @mtd: MTD block structure + * @chip: NAND chip object * @buf: raw data read from the chip * @read_ecc: ECC from the chip * @calc_ecc: the ECC calculated from raw data * * Detect and correct bit errors for a data byte block */ -int nand_bch_correct_data(struct mtd_info *mtd, unsigned char *buf, +int nand_bch_correct_data(struct nand_chip *chip, unsigned char *buf, unsigned char *read_ecc, unsigned char *calc_ecc) { - const struct nand_chip *chip = mtd_to_nand(mtd); struct nand_bch_control *nbc = chip->ecc.priv; unsigned int *errloc = nbc->errloc; int i, count; diff --git a/drivers/mtd/nand/raw/nand_ecc.c b/drivers/mtd/nand/raw/nand_ecc.c index 8e132edbc5ce..4f4347533058 100644 --- a/drivers/mtd/nand/raw/nand_ecc.c +++ b/drivers/mtd/nand/raw/nand_ecc.c @@ -132,9 +132,10 @@ static const char addressbits[256] = { * @buf: input buffer with raw data * @eccsize: data bytes per ECC step (256 or 512) * @code: output buffer with ECC + * @sm_order: Smart Media byte ordering */ void __nand_calculate_ecc(const unsigned char *buf, unsigned int eccsize, - unsigned char *code) + unsigned char *code, bool sm_order) { int i; const uint32_t *bp = (uint32_t *)buf; @@ -330,45 +331,26 @@ void __nand_calculate_ecc(const unsigned char *buf, unsigned int eccsize, * possible, but benchmarks showed that on the system this is developed * the code below is the fastest */ -#ifdef CONFIG_MTD_NAND_ECC_SMC - code[0] = - (invparity[rp7] << 7) | - (invparity[rp6] << 6) | - (invparity[rp5] << 5) | - (invparity[rp4] << 4) | - (invparity[rp3] << 3) | - (invparity[rp2] << 2) | - (invparity[rp1] << 1) | - (invparity[rp0]); - code[1] = - (invparity[rp15] << 7) | - (invparity[rp14] << 6) | - (invparity[rp13] << 5) | - (invparity[rp12] << 4) | - (invparity[rp11] << 3) | - (invparity[rp10] << 2) | - (invparity[rp9] << 1) | - (invparity[rp8]); -#else - code[1] = - (invparity[rp7] << 7) | - (invparity[rp6] << 6) | - (invparity[rp5] << 5) | - (invparity[rp4] << 4) | - (invparity[rp3] << 3) | - (invparity[rp2] << 2) | - (invparity[rp1] << 1) | - (invparity[rp0]); - code[0] = - (invparity[rp15] << 7) | - (invparity[rp14] << 6) | - (invparity[rp13] << 5) | - (invparity[rp12] << 4) | - (invparity[rp11] << 3) | - (invparity[rp10] << 2) | - (invparity[rp9] << 1) | - (invparity[rp8]); -#endif + if (sm_order) { + code[0] = (invparity[rp7] << 7) | (invparity[rp6] << 6) | + (invparity[rp5] << 5) | (invparity[rp4] << 4) | + (invparity[rp3] << 3) | (invparity[rp2] << 2) | + (invparity[rp1] << 1) | (invparity[rp0]); + code[1] = (invparity[rp15] << 7) | (invparity[rp14] << 6) | + (invparity[rp13] << 5) | (invparity[rp12] << 4) | + (invparity[rp11] << 3) | (invparity[rp10] << 2) | + (invparity[rp9] << 1) | (invparity[rp8]); + } else { + code[1] = (invparity[rp7] << 7) | (invparity[rp6] << 6) | + (invparity[rp5] << 5) | (invparity[rp4] << 4) | + (invparity[rp3] << 3) | (invparity[rp2] << 2) | + (invparity[rp1] << 1) | (invparity[rp0]); + code[0] = (invparity[rp15] << 7) | (invparity[rp14] << 6) | + (invparity[rp13] << 5) | (invparity[rp12] << 4) | + (invparity[rp11] << 3) | (invparity[rp10] << 2) | + (invparity[rp9] << 1) | (invparity[rp8]); + } + if (eccsize_mult == 1) code[2] = (invparity[par & 0xf0] << 7) | @@ -394,15 +376,16 @@ EXPORT_SYMBOL(__nand_calculate_ecc); /** * nand_calculate_ecc - [NAND Interface] Calculate 3-byte ECC for 256/512-byte * block - * @mtd: MTD block structure + * @chip: NAND chip object * @buf: input buffer with raw data * @code: output buffer with ECC */ -int nand_calculate_ecc(struct mtd_info *mtd, const unsigned char *buf, +int nand_calculate_ecc(struct nand_chip *chip, const unsigned char *buf, unsigned char *code) { - __nand_calculate_ecc(buf, - mtd_to_nand(mtd)->ecc.size, code); + bool sm_order = chip->ecc.options & NAND_ECC_SOFT_HAMMING_SM_ORDER; + + __nand_calculate_ecc(buf, chip->ecc.size, code, sm_order); return 0; } @@ -414,12 +397,13 @@ EXPORT_SYMBOL(nand_calculate_ecc); * @read_ecc: ECC from the chip * @calc_ecc: the ECC calculated from raw data * @eccsize: data bytes per ECC step (256 or 512) + * @sm_order: Smart Media byte order * * Detect and correct a 1 bit error for eccsize byte block */ int __nand_correct_data(unsigned char *buf, unsigned char *read_ecc, unsigned char *calc_ecc, - unsigned int eccsize) + unsigned int eccsize, bool sm_order) { unsigned char b0, b1, b2, bit_addr; unsigned int byte_addr; @@ -431,13 +415,14 @@ int __nand_correct_data(unsigned char *buf, * we might need the xor result more than once, * so keep them in a local var */ -#ifdef CONFIG_MTD_NAND_ECC_SMC - b0 = read_ecc[0] ^ calc_ecc[0]; - b1 = read_ecc[1] ^ calc_ecc[1]; -#else - b0 = read_ecc[1] ^ calc_ecc[1]; - b1 = read_ecc[0] ^ calc_ecc[0]; -#endif + if (sm_order) { + b0 = read_ecc[0] ^ calc_ecc[0]; + b1 = read_ecc[1] ^ calc_ecc[1]; + } else { + b0 = read_ecc[1] ^ calc_ecc[1]; + b1 = read_ecc[0] ^ calc_ecc[0]; + } + b2 = read_ecc[2] ^ calc_ecc[2]; /* check if there are any bitfaults */ @@ -491,18 +476,20 @@ EXPORT_SYMBOL(__nand_correct_data); /** * nand_correct_data - [NAND Interface] Detect and correct bit error(s) - * @mtd: MTD block structure + * @chip: NAND chip object * @buf: raw data read from the chip * @read_ecc: ECC from the chip * @calc_ecc: the ECC calculated from raw data * * Detect and correct a 1 bit error for 256/512 byte block */ -int nand_correct_data(struct mtd_info *mtd, unsigned char *buf, +int nand_correct_data(struct nand_chip *chip, unsigned char *buf, unsigned char *read_ecc, unsigned char *calc_ecc) { - return __nand_correct_data(buf, read_ecc, calc_ecc, - mtd_to_nand(mtd)->ecc.size); + bool sm_order = chip->ecc.options & NAND_ECC_SOFT_HAMMING_SM_ORDER; + + return __nand_correct_data(buf, read_ecc, calc_ecc, chip->ecc.size, + sm_order); } EXPORT_SYMBOL(nand_correct_data); diff --git a/drivers/mtd/nand/raw/nand_esmt.c b/drivers/mtd/nand/raw/nand_esmt.c new file mode 100644 index 000000000000..96f039a83bc8 --- /dev/null +++ b/drivers/mtd/nand/raw/nand_esmt.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2018 Toradex AG + * + * Author: Marcel Ziswiler <marcel.ziswiler@toradex.com> + */ + +#include <linux/mtd/rawnand.h> +#include "internals.h" + +static void esmt_nand_decode_id(struct nand_chip *chip) +{ + nand_decode_ext_id(chip); + + /* Extract ECC requirements from 5th id byte. */ + if (chip->id.len >= 5 && nand_is_slc(chip)) { + chip->ecc_step_ds = 512; + switch (chip->id.data[4] & 0x3) { + case 0x0: + chip->ecc_strength_ds = 4; + break; + case 0x1: + chip->ecc_strength_ds = 2; + break; + case 0x2: + chip->ecc_strength_ds = 1; + break; + default: + WARN(1, "Could not get ECC info"); + chip->ecc_step_ds = 0; + break; + } + } +} + +static int esmt_nand_init(struct nand_chip *chip) +{ + if (nand_is_slc(chip)) + chip->bbt_options |= NAND_BBT_SCAN2NDPAGE; + + return 0; +} + +const struct nand_manufacturer_ops esmt_nand_manuf_ops = { + .detect = esmt_nand_decode_id, + .init = esmt_nand_init, +}; diff --git a/drivers/mtd/nand/raw/nand_hynix.c b/drivers/mtd/nand/raw/nand_hynix.c index 4ffbb26e76d6..ac1b5c103968 100644 --- a/drivers/mtd/nand/raw/nand_hynix.c +++ b/drivers/mtd/nand/raw/nand_hynix.c @@ -15,10 +15,11 @@ * GNU General Public License for more details. */ -#include <linux/mtd/rawnand.h> #include <linux/sizes.h> #include <linux/slab.h> +#include "internals.h" + #define NAND_HYNIX_CMD_SET_PARAMS 0x36 #define NAND_HYNIX_CMD_APPLY_PARAMS 0x16 @@ -79,8 +80,6 @@ static bool hynix_nand_has_valid_jedecid(struct nand_chip *chip) static int hynix_nand_cmd_op(struct nand_chip *chip, u8 cmd) { - struct mtd_info *mtd = nand_to_mtd(chip); - if (chip->exec_op) { struct nand_op_instr instrs[] = { NAND_OP_CMD(cmd, 0), @@ -90,14 +89,13 @@ static int hynix_nand_cmd_op(struct nand_chip *chip, u8 cmd) return nand_exec_op(chip, &op); } - chip->cmdfunc(mtd, cmd, -1, -1); + chip->legacy.cmdfunc(chip, cmd, -1, -1); return 0; } static int hynix_nand_reg_write_op(struct nand_chip *chip, u8 addr, u8 val) { - struct mtd_info *mtd = nand_to_mtd(chip); u16 column = ((u16)addr << 8) | addr; if (chip->exec_op) { @@ -110,15 +108,14 @@ static int hynix_nand_reg_write_op(struct nand_chip *chip, u8 addr, u8 val) return nand_exec_op(chip, &op); } - chip->cmdfunc(mtd, NAND_CMD_NONE, column, -1); - chip->write_byte(mtd, val); + chip->legacy.cmdfunc(chip, NAND_CMD_NONE, column, -1); + chip->legacy.write_byte(chip, val); return 0; } -static int hynix_nand_setup_read_retry(struct mtd_info *mtd, int retry_mode) +static int hynix_nand_setup_read_retry(struct nand_chip *chip, int retry_mode) { - struct nand_chip *chip = mtd_to_nand(mtd); struct hynix_nand *hynix = nand_get_manufacturer_data(chip); const u8 *values; int i, ret; diff --git a/drivers/mtd/nand/raw/nand_ids.c b/drivers/mtd/nand/raw/nand_ids.c index 5423c3bb388e..ea5a342cd91e 100644 --- a/drivers/mtd/nand/raw/nand_ids.c +++ b/drivers/mtd/nand/raw/nand_ids.c @@ -6,9 +6,11 @@ * published by the Free Software Foundation. * */ -#include <linux/mtd/rawnand.h> + #include <linux/sizes.h> +#include "internals.h" + #define LP_OPTIONS 0 #define LP_OPTIONS16 (LP_OPTIONS | NAND_BUSWIDTH_16) @@ -169,21 +171,21 @@ struct nand_flash_dev nand_flash_ids[] = { /* Manufacturer IDs */ static const struct nand_manufacturer nand_manufacturers[] = { - {NAND_MFR_TOSHIBA, "Toshiba", &toshiba_nand_manuf_ops}, - {NAND_MFR_ESMT, "ESMT"}, - {NAND_MFR_SAMSUNG, "Samsung", &samsung_nand_manuf_ops}, + {NAND_MFR_AMD, "AMD/Spansion", &amd_nand_manuf_ops}, + {NAND_MFR_ATO, "ATO"}, + {NAND_MFR_EON, "Eon"}, + {NAND_MFR_ESMT, "ESMT", &esmt_nand_manuf_ops}, {NAND_MFR_FUJITSU, "Fujitsu"}, - {NAND_MFR_NATIONAL, "National"}, - {NAND_MFR_RENESAS, "Renesas"}, - {NAND_MFR_STMICRO, "ST Micro"}, {NAND_MFR_HYNIX, "Hynix", &hynix_nand_manuf_ops}, - {NAND_MFR_MICRON, "Micron", µn_nand_manuf_ops}, - {NAND_MFR_AMD, "AMD/Spansion", &amd_nand_manuf_ops}, + {NAND_MFR_INTEL, "Intel"}, {NAND_MFR_MACRONIX, "Macronix", ¯onix_nand_manuf_ops}, - {NAND_MFR_EON, "Eon"}, + {NAND_MFR_MICRON, "Micron", µn_nand_manuf_ops}, + {NAND_MFR_NATIONAL, "National"}, + {NAND_MFR_RENESAS, "Renesas"}, + {NAND_MFR_SAMSUNG, "Samsung", &samsung_nand_manuf_ops}, {NAND_MFR_SANDISK, "SanDisk"}, - {NAND_MFR_INTEL, "Intel"}, - {NAND_MFR_ATO, "ATO"}, + {NAND_MFR_STMICRO, "ST Micro"}, + {NAND_MFR_TOSHIBA, "Toshiba", &toshiba_nand_manuf_ops}, {NAND_MFR_WINBOND, "Winbond"}, }; diff --git a/drivers/mtd/nand/raw/nand_jedec.c b/drivers/mtd/nand/raw/nand_jedec.c new file mode 100644 index 000000000000..5c26492c841d --- /dev/null +++ b/drivers/mtd/nand/raw/nand_jedec.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2000 Steven J. Hill (sjhill@realitydiluted.com) + * 2002-2006 Thomas Gleixner (tglx@linutronix.de) + * + * Credits: + * David Woodhouse for adding multichip support + * + * Aleph One Ltd. and Toby Churchill Ltd. for supporting the + * rework for 2K page size chips + * + * This file contains all ONFI helpers. + */ + +#include <linux/slab.h> + +#include "internals.h" + +/* + * Check if the NAND chip is JEDEC compliant, returns 1 if it is, 0 otherwise. + */ +int nand_jedec_detect(struct nand_chip *chip) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + struct nand_jedec_params *p; + struct jedec_ecc_info *ecc; + int jedec_version = 0; + char id[5]; + int i, val, ret; + + /* Try JEDEC for unknown chip or LP */ + ret = nand_readid_op(chip, 0x40, id, sizeof(id)); + if (ret || strncmp(id, "JEDEC", sizeof(id))) + return 0; + + /* JEDEC chip: allocate a buffer to hold its parameter page */ + p = kzalloc(sizeof(*p), GFP_KERNEL); + if (!p) + return -ENOMEM; + + ret = nand_read_param_page_op(chip, 0x40, NULL, 0); + if (ret) { + ret = 0; + goto free_jedec_param_page; + } + + for (i = 0; i < 3; i++) { + ret = nand_read_data_op(chip, p, sizeof(*p), true); + if (ret) { + ret = 0; + goto free_jedec_param_page; + } + + if (onfi_crc16(ONFI_CRC_BASE, (uint8_t *)p, 510) == + le16_to_cpu(p->crc)) + break; + } + + if (i == 3) { + pr_err("Could not find valid JEDEC parameter page; aborting\n"); + goto free_jedec_param_page; + } + + /* Check version */ + val = le16_to_cpu(p->revision); + if (val & (1 << 2)) + jedec_version = 10; + else if (val & (1 << 1)) + jedec_version = 1; /* vendor specific version */ + + if (!jedec_version) { + pr_info("unsupported JEDEC version: %d\n", val); + goto free_jedec_param_page; + } + + sanitize_string(p->manufacturer, sizeof(p->manufacturer)); + sanitize_string(p->model, sizeof(p->model)); + chip->parameters.model = kstrdup(p->model, GFP_KERNEL); + if (!chip->parameters.model) { + ret = -ENOMEM; + goto free_jedec_param_page; + } + + mtd->writesize = le32_to_cpu(p->byte_per_page); + + /* Please reference to the comment for nand_flash_detect_onfi. */ + mtd->erasesize = 1 << (fls(le32_to_cpu(p->pages_per_block)) - 1); + mtd->erasesize *= mtd->writesize; + + mtd->oobsize = le16_to_cpu(p->spare_bytes_per_page); + + /* Please reference to the comment for nand_flash_detect_onfi. */ + chip->chipsize = 1 << (fls(le32_to_cpu(p->blocks_per_lun)) - 1); + chip->chipsize *= (uint64_t)mtd->erasesize * p->lun_count; + chip->bits_per_cell = p->bits_per_cell; + + if (le16_to_cpu(p->features) & JEDEC_FEATURE_16_BIT_BUS) + chip->options |= NAND_BUSWIDTH_16; + + /* ECC info */ + ecc = &p->ecc_info[0]; + + if (ecc->codeword_size >= 9) { + chip->ecc_strength_ds = ecc->ecc_bits; + chip->ecc_step_ds = 1 << ecc->codeword_size; + } else { + pr_warn("Invalid codeword size\n"); + } + +free_jedec_param_page: + kfree(p); + return ret; +} diff --git a/drivers/mtd/nand/raw/nand_legacy.c b/drivers/mtd/nand/raw/nand_legacy.c new file mode 100644 index 000000000000..c5ddc86cd98c --- /dev/null +++ b/drivers/mtd/nand/raw/nand_legacy.c @@ -0,0 +1,642 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2000 Steven J. Hill (sjhill@realitydiluted.com) + * 2002-2006 Thomas Gleixner (tglx@linutronix.de) + * + * Credits: + * David Woodhouse for adding multichip support + * + * Aleph One Ltd. and Toby Churchill Ltd. for supporting the + * rework for 2K page size chips + * + * This file contains all legacy helpers/code that should be removed + * at some point. + */ + +#include <linux/delay.h> +#include <linux/io.h> +#include <linux/nmi.h> + +#include "internals.h" + +/** + * nand_read_byte - [DEFAULT] read one byte from the chip + * @chip: NAND chip object + * + * Default read function for 8bit buswidth + */ +static uint8_t nand_read_byte(struct nand_chip *chip) +{ + return readb(chip->legacy.IO_ADDR_R); +} + +/** + * nand_read_byte16 - [DEFAULT] read one byte endianness aware from the chip + * @chip: NAND chip object + * + * Default read function for 16bit buswidth with endianness conversion. + * + */ +static uint8_t nand_read_byte16(struct nand_chip *chip) +{ + return (uint8_t) cpu_to_le16(readw(chip->legacy.IO_ADDR_R)); +} + +/** + * nand_select_chip - [DEFAULT] control CE line + * @chip: NAND chip object + * @chipnr: chipnumber to select, -1 for deselect + * + * Default select function for 1 chip devices. + */ +static void nand_select_chip(struct nand_chip *chip, int chipnr) +{ + switch (chipnr) { + case -1: + chip->legacy.cmd_ctrl(chip, NAND_CMD_NONE, + 0 | NAND_CTRL_CHANGE); + break; + case 0: + break; + + default: + BUG(); + } +} + +/** + * nand_write_byte - [DEFAULT] write single byte to chip + * @chip: NAND chip object + * @byte: value to write + * + * Default function to write a byte to I/O[7:0] + */ +static void nand_write_byte(struct nand_chip *chip, uint8_t byte) +{ + chip->legacy.write_buf(chip, &byte, 1); +} + +/** + * nand_write_byte16 - [DEFAULT] write single byte to a chip with width 16 + * @chip: NAND chip object + * @byte: value to write + * + * Default function to write a byte to I/O[7:0] on a 16-bit wide chip. + */ +static void nand_write_byte16(struct nand_chip *chip, uint8_t byte) +{ + uint16_t word = byte; + + /* + * It's not entirely clear what should happen to I/O[15:8] when writing + * a byte. The ONFi spec (Revision 3.1; 2012-09-19, Section 2.16) reads: + * + * When the host supports a 16-bit bus width, only data is + * transferred at the 16-bit width. All address and command line + * transfers shall use only the lower 8-bits of the data bus. During + * command transfers, the host may place any value on the upper + * 8-bits of the data bus. During address transfers, the host shall + * set the upper 8-bits of the data bus to 00h. + * + * One user of the write_byte callback is nand_set_features. The + * four parameters are specified to be written to I/O[7:0], but this is + * neither an address nor a command transfer. Let's assume a 0 on the + * upper I/O lines is OK. + */ + chip->legacy.write_buf(chip, (uint8_t *)&word, 2); +} + +/** + * nand_write_buf - [DEFAULT] write buffer to chip + * @chip: NAND chip object + * @buf: data buffer + * @len: number of bytes to write + * + * Default write function for 8bit buswidth. + */ +static void nand_write_buf(struct nand_chip *chip, const uint8_t *buf, int len) +{ + iowrite8_rep(chip->legacy.IO_ADDR_W, buf, len); +} + +/** + * nand_read_buf - [DEFAULT] read chip data into buffer + * @chip: NAND chip object + * @buf: buffer to store date + * @len: number of bytes to read + * + * Default read function for 8bit buswidth. + */ +static void nand_read_buf(struct nand_chip *chip, uint8_t *buf, int len) +{ + ioread8_rep(chip->legacy.IO_ADDR_R, buf, len); +} + +/** + * nand_write_buf16 - [DEFAULT] write buffer to chip + * @chip: NAND chip object + * @buf: data buffer + * @len: number of bytes to write + * + * Default write function for 16bit buswidth. + */ +static void nand_write_buf16(struct nand_chip *chip, const uint8_t *buf, + int len) +{ + u16 *p = (u16 *) buf; + + iowrite16_rep(chip->legacy.IO_ADDR_W, p, len >> 1); +} + +/** + * nand_read_buf16 - [DEFAULT] read chip data into buffer + * @chip: NAND chip object + * @buf: buffer to store date + * @len: number of bytes to read + * + * Default read function for 16bit buswidth. + */ +static void nand_read_buf16(struct nand_chip *chip, uint8_t *buf, int len) +{ + u16 *p = (u16 *) buf; + + ioread16_rep(chip->legacy.IO_ADDR_R, p, len >> 1); +} + +/** + * panic_nand_wait_ready - [GENERIC] Wait for the ready pin after commands. + * @mtd: MTD device structure + * @timeo: Timeout + * + * Helper function for nand_wait_ready used when needing to wait in interrupt + * context. + */ +static void panic_nand_wait_ready(struct mtd_info *mtd, unsigned long timeo) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + int i; + + /* Wait for the device to get ready */ + for (i = 0; i < timeo; i++) { + if (chip->legacy.dev_ready(chip)) + break; + touch_softlockup_watchdog(); + mdelay(1); + } +} + +/** + * nand_wait_ready - [GENERIC] Wait for the ready pin after commands. + * @chip: NAND chip object + * + * Wait for the ready pin after a command, and warn if a timeout occurs. + */ +void nand_wait_ready(struct nand_chip *chip) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + unsigned long timeo = 400; + + if (in_interrupt() || oops_in_progress) + return panic_nand_wait_ready(mtd, timeo); + + /* Wait until command is processed or timeout occurs */ + timeo = jiffies + msecs_to_jiffies(timeo); + do { + if (chip->legacy.dev_ready(chip)) + return; + cond_resched(); + } while (time_before(jiffies, timeo)); + + if (!chip->legacy.dev_ready(chip)) + pr_warn_ratelimited("timeout while waiting for chip to become ready\n"); +} +EXPORT_SYMBOL_GPL(nand_wait_ready); + +/** + * nand_wait_status_ready - [GENERIC] Wait for the ready status after commands. + * @mtd: MTD device structure + * @timeo: Timeout in ms + * + * Wait for status ready (i.e. command done) or timeout. + */ +static void nand_wait_status_ready(struct mtd_info *mtd, unsigned long timeo) +{ + register struct nand_chip *chip = mtd_to_nand(mtd); + int ret; + + timeo = jiffies + msecs_to_jiffies(timeo); + do { + u8 status; + + ret = nand_read_data_op(chip, &status, sizeof(status), true); + if (ret) + return; + + if (status & NAND_STATUS_READY) + break; + touch_softlockup_watchdog(); + } while (time_before(jiffies, timeo)); +}; + +/** + * nand_command - [DEFAULT] Send command to NAND device + * @chip: NAND chip object + * @command: the command to be sent + * @column: the column address for this command, -1 if none + * @page_addr: the page address for this command, -1 if none + * + * Send command to NAND device. This function is used for small page devices + * (512 Bytes per page). + */ +static void nand_command(struct nand_chip *chip, unsigned int command, + int column, int page_addr) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + int ctrl = NAND_CTRL_CLE | NAND_CTRL_CHANGE; + + /* Write out the command to the device */ + if (command == NAND_CMD_SEQIN) { + int readcmd; + + if (column >= mtd->writesize) { + /* OOB area */ + column -= mtd->writesize; + readcmd = NAND_CMD_READOOB; + } else if (column < 256) { + /* First 256 bytes --> READ0 */ + readcmd = NAND_CMD_READ0; + } else { + column -= 256; + readcmd = NAND_CMD_READ1; + } + chip->legacy.cmd_ctrl(chip, readcmd, ctrl); + ctrl &= ~NAND_CTRL_CHANGE; + } + if (command != NAND_CMD_NONE) + chip->legacy.cmd_ctrl(chip, command, ctrl); + + /* Address cycle, when necessary */ + ctrl = NAND_CTRL_ALE | NAND_CTRL_CHANGE; + /* Serially input address */ + if (column != -1) { + /* Adjust columns for 16 bit buswidth */ + if (chip->options & NAND_BUSWIDTH_16 && + !nand_opcode_8bits(command)) + column >>= 1; + chip->legacy.cmd_ctrl(chip, column, ctrl); + ctrl &= ~NAND_CTRL_CHANGE; + } + if (page_addr != -1) { + chip->legacy.cmd_ctrl(chip, page_addr, ctrl); + ctrl &= ~NAND_CTRL_CHANGE; + chip->legacy.cmd_ctrl(chip, page_addr >> 8, ctrl); + if (chip->options & NAND_ROW_ADDR_3) + chip->legacy.cmd_ctrl(chip, page_addr >> 16, ctrl); + } + chip->legacy.cmd_ctrl(chip, NAND_CMD_NONE, + NAND_NCE | NAND_CTRL_CHANGE); + + /* + * Program and erase have their own busy handlers status and sequential + * in needs no delay + */ + switch (command) { + + case NAND_CMD_NONE: + case NAND_CMD_PAGEPROG: + case NAND_CMD_ERASE1: + case NAND_CMD_ERASE2: + case NAND_CMD_SEQIN: + case NAND_CMD_STATUS: + case NAND_CMD_READID: + case NAND_CMD_SET_FEATURES: + return; + + case NAND_CMD_RESET: + if (chip->legacy.dev_ready) + break; + udelay(chip->legacy.chip_delay); + chip->legacy.cmd_ctrl(chip, NAND_CMD_STATUS, + NAND_CTRL_CLE | NAND_CTRL_CHANGE); + chip->legacy.cmd_ctrl(chip, NAND_CMD_NONE, + NAND_NCE | NAND_CTRL_CHANGE); + /* EZ-NAND can take upto 250ms as per ONFi v4.0 */ + nand_wait_status_ready(mtd, 250); + return; + + /* This applies to read commands */ + case NAND_CMD_READ0: + /* + * READ0 is sometimes used to exit GET STATUS mode. When this + * is the case no address cycles are requested, and we can use + * this information to detect that we should not wait for the + * device to be ready. + */ + if (column == -1 && page_addr == -1) + return; + + default: + /* + * If we don't have access to the busy pin, we apply the given + * command delay + */ + if (!chip->legacy.dev_ready) { + udelay(chip->legacy.chip_delay); + return; + } + } + /* + * Apply this short delay always to ensure that we do wait tWB in + * any case on any machine. + */ + ndelay(100); + + nand_wait_ready(chip); +} + +static void nand_ccs_delay(struct nand_chip *chip) +{ + /* + * The controller already takes care of waiting for tCCS when the RNDIN + * or RNDOUT command is sent, return directly. + */ + if (!(chip->options & NAND_WAIT_TCCS)) + return; + + /* + * Wait tCCS_min if it is correctly defined, otherwise wait 500ns + * (which should be safe for all NANDs). + */ + if (chip->setup_data_interface) + ndelay(chip->data_interface.timings.sdr.tCCS_min / 1000); + else + ndelay(500); +} + +/** + * nand_command_lp - [DEFAULT] Send command to NAND large page device + * @chip: NAND chip object + * @command: the command to be sent + * @column: the column address for this command, -1 if none + * @page_addr: the page address for this command, -1 if none + * + * Send command to NAND device. This is the version for the new large page + * devices. We don't have the separate regions as we have in the small page + * devices. We must emulate NAND_CMD_READOOB to keep the code compatible. + */ +static void nand_command_lp(struct nand_chip *chip, unsigned int command, + int column, int page_addr) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + + /* Emulate NAND_CMD_READOOB */ + if (command == NAND_CMD_READOOB) { + column += mtd->writesize; + command = NAND_CMD_READ0; + } + + /* Command latch cycle */ + if (command != NAND_CMD_NONE) + chip->legacy.cmd_ctrl(chip, command, + NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE); + + if (column != -1 || page_addr != -1) { + int ctrl = NAND_CTRL_CHANGE | NAND_NCE | NAND_ALE; + + /* Serially input address */ + if (column != -1) { + /* Adjust columns for 16 bit buswidth */ + if (chip->options & NAND_BUSWIDTH_16 && + !nand_opcode_8bits(command)) + column >>= 1; + chip->legacy.cmd_ctrl(chip, column, ctrl); + ctrl &= ~NAND_CTRL_CHANGE; + + /* Only output a single addr cycle for 8bits opcodes. */ + if (!nand_opcode_8bits(command)) + chip->legacy.cmd_ctrl(chip, column >> 8, ctrl); + } + if (page_addr != -1) { + chip->legacy.cmd_ctrl(chip, page_addr, ctrl); + chip->legacy.cmd_ctrl(chip, page_addr >> 8, + NAND_NCE | NAND_ALE); + if (chip->options & NAND_ROW_ADDR_3) + chip->legacy.cmd_ctrl(chip, page_addr >> 16, + NAND_NCE | NAND_ALE); + } + } + chip->legacy.cmd_ctrl(chip, NAND_CMD_NONE, + NAND_NCE | NAND_CTRL_CHANGE); + + /* + * Program and erase have their own busy handlers status, sequential + * in and status need no delay. + */ + switch (command) { + + case NAND_CMD_NONE: + case NAND_CMD_CACHEDPROG: + case NAND_CMD_PAGEPROG: + case NAND_CMD_ERASE1: + case NAND_CMD_ERASE2: + case NAND_CMD_SEQIN: + case NAND_CMD_STATUS: + case NAND_CMD_READID: + case NAND_CMD_SET_FEATURES: + return; + + case NAND_CMD_RNDIN: + nand_ccs_delay(chip); + return; + + case NAND_CMD_RESET: + if (chip->legacy.dev_ready) + break; + udelay(chip->legacy.chip_delay); + chip->legacy.cmd_ctrl(chip, NAND_CMD_STATUS, + NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE); + chip->legacy.cmd_ctrl(chip, NAND_CMD_NONE, + NAND_NCE | NAND_CTRL_CHANGE); + /* EZ-NAND can take upto 250ms as per ONFi v4.0 */ + nand_wait_status_ready(mtd, 250); + return; + + case NAND_CMD_RNDOUT: + /* No ready / busy check necessary */ + chip->legacy.cmd_ctrl(chip, NAND_CMD_RNDOUTSTART, + NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE); + chip->legacy.cmd_ctrl(chip, NAND_CMD_NONE, + NAND_NCE | NAND_CTRL_CHANGE); + + nand_ccs_delay(chip); + return; + + case NAND_CMD_READ0: + /* + * READ0 is sometimes used to exit GET STATUS mode. When this + * is the case no address cycles are requested, and we can use + * this information to detect that READSTART should not be + * issued. + */ + if (column == -1 && page_addr == -1) + return; + + chip->legacy.cmd_ctrl(chip, NAND_CMD_READSTART, + NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE); + chip->legacy.cmd_ctrl(chip, NAND_CMD_NONE, + NAND_NCE | NAND_CTRL_CHANGE); + + /* This applies to read commands */ + default: + /* + * If we don't have access to the busy pin, we apply the given + * command delay. + */ + if (!chip->legacy.dev_ready) { + udelay(chip->legacy.chip_delay); + return; + } + } + + /* + * Apply this short delay always to ensure that we do wait tWB in + * any case on any machine. + */ + ndelay(100); + + nand_wait_ready(chip); +} + +/** + * nand_get_set_features_notsupp - set/get features stub returning -ENOTSUPP + * @chip: nand chip info structure + * @addr: feature address. + * @subfeature_param: the subfeature parameters, a four bytes array. + * + * Should be used by NAND controller drivers that do not support the SET/GET + * FEATURES operations. + */ +int nand_get_set_features_notsupp(struct nand_chip *chip, int addr, + u8 *subfeature_param) +{ + return -ENOTSUPP; +} +EXPORT_SYMBOL(nand_get_set_features_notsupp); + +/** + * nand_wait - [DEFAULT] wait until the command is done + * @mtd: MTD device structure + * @chip: NAND chip structure + * + * Wait for command done. This applies to erase and program only. + */ +static int nand_wait(struct nand_chip *chip) +{ + + unsigned long timeo = 400; + u8 status; + int ret; + + /* + * Apply this short delay always to ensure that we do wait tWB in any + * case on any machine. + */ + ndelay(100); + + ret = nand_status_op(chip, NULL); + if (ret) + return ret; + + if (in_interrupt() || oops_in_progress) + panic_nand_wait(chip, timeo); + else { + timeo = jiffies + msecs_to_jiffies(timeo); + do { + if (chip->legacy.dev_ready) { + if (chip->legacy.dev_ready(chip)) + break; + } else { + ret = nand_read_data_op(chip, &status, + sizeof(status), true); + if (ret) + return ret; + + if (status & NAND_STATUS_READY) + break; + } + cond_resched(); + } while (time_before(jiffies, timeo)); + } + + ret = nand_read_data_op(chip, &status, sizeof(status), true); + if (ret) + return ret; + + /* This can happen if in case of timeout or buggy dev_ready */ + WARN_ON(!(status & NAND_STATUS_READY)); + return status; +} + +void nand_legacy_set_defaults(struct nand_chip *chip) +{ + unsigned int busw = chip->options & NAND_BUSWIDTH_16; + + if (chip->exec_op) + return; + + /* check for proper chip_delay setup, set 20us if not */ + if (!chip->legacy.chip_delay) + chip->legacy.chip_delay = 20; + + /* check, if a user supplied command function given */ + if (!chip->legacy.cmdfunc && !chip->exec_op) + chip->legacy.cmdfunc = nand_command; + + /* check, if a user supplied wait function given */ + if (chip->legacy.waitfunc == NULL) + chip->legacy.waitfunc = nand_wait; + + if (!chip->select_chip) + chip->select_chip = nand_select_chip; + + /* If called twice, pointers that depend on busw may need to be reset */ + if (!chip->legacy.read_byte || chip->legacy.read_byte == nand_read_byte) + chip->legacy.read_byte = busw ? nand_read_byte16 : nand_read_byte; + if (!chip->legacy.write_buf || chip->legacy.write_buf == nand_write_buf) + chip->legacy.write_buf = busw ? nand_write_buf16 : nand_write_buf; + if (!chip->legacy.write_byte || chip->legacy.write_byte == nand_write_byte) + chip->legacy.write_byte = busw ? nand_write_byte16 : nand_write_byte; + if (!chip->legacy.read_buf || chip->legacy.read_buf == nand_read_buf) + chip->legacy.read_buf = busw ? nand_read_buf16 : nand_read_buf; +} + +void nand_legacy_adjust_cmdfunc(struct nand_chip *chip) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + + /* Do not replace user supplied command function! */ + if (mtd->writesize > 512 && chip->legacy.cmdfunc == nand_command) + chip->legacy.cmdfunc = nand_command_lp; +} + +int nand_legacy_check_hooks(struct nand_chip *chip) +{ + /* + * ->legacy.cmdfunc() is legacy and will only be used if ->exec_op() is + * not populated. + */ + if (chip->exec_op) + return 0; + + /* + * Default functions assigned for ->legacy.cmdfunc() and + * ->select_chip() both expect ->legacy.cmd_ctrl() to be populated. + */ + if ((!chip->legacy.cmdfunc || !chip->select_chip) && + !chip->legacy.cmd_ctrl) { + pr_err("->legacy.cmd_ctrl() should be provided\n"); + return -EINVAL; + } + + return 0; +} diff --git a/drivers/mtd/nand/raw/nand_macronix.c b/drivers/mtd/nand/raw/nand_macronix.c index 49c546c97c6f..358dcc957bb2 100644 --- a/drivers/mtd/nand/raw/nand_macronix.c +++ b/drivers/mtd/nand/raw/nand_macronix.c @@ -15,7 +15,7 @@ * GNU General Public License for more details. */ -#include <linux/mtd/rawnand.h> +#include "internals.h" /* * Macronix AC series does not support using SET/GET_FEATURES to change diff --git a/drivers/mtd/nand/raw/nand_micron.c b/drivers/mtd/nand/raw/nand_micron.c index f5dc0a7a2456..b85e1c13b79e 100644 --- a/drivers/mtd/nand/raw/nand_micron.c +++ b/drivers/mtd/nand/raw/nand_micron.c @@ -15,9 +15,10 @@ * GNU General Public License for more details. */ -#include <linux/mtd/rawnand.h> #include <linux/slab.h> +#include "internals.h" + /* * Special Micron status bit 3 indicates that the block has been * corrected by on-die ECC and should be rewritten. @@ -74,9 +75,8 @@ struct micron_nand { struct micron_on_die_ecc ecc; }; -static int micron_nand_setup_read_retry(struct mtd_info *mtd, int retry_mode) +static int micron_nand_setup_read_retry(struct nand_chip *chip, int retry_mode) { - struct nand_chip *chip = mtd_to_nand(mtd); u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = {retry_mode}; return nand_set_features(chip, ONFI_FEATURE_ADDR_READ_RETRY, feature); @@ -290,10 +290,10 @@ static int micron_nand_on_die_ecc_status_8(struct nand_chip *chip, u8 status) } static int -micron_nand_read_page_on_die_ecc(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, - int page) +micron_nand_read_page_on_die_ecc(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); u8 status; int ret, max_bitflips = 0; @@ -332,9 +332,8 @@ out: } static int -micron_nand_write_page_on_die_ecc(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, - int page) +micron_nand_write_page_on_die_ecc(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { int ret; @@ -342,7 +341,7 @@ micron_nand_write_page_on_die_ecc(struct mtd_info *mtd, struct nand_chip *chip, if (ret) return ret; - ret = nand_write_page_raw(mtd, chip, buf, oob_required, page); + ret = nand_write_page_raw(chip, buf, oob_required, page); micron_nand_on_die_ecc_setup(chip, false); return ret; diff --git a/drivers/mtd/nand/raw/nand_onfi.c b/drivers/mtd/nand/raw/nand_onfi.c new file mode 100644 index 000000000000..d8184cf591ad --- /dev/null +++ b/drivers/mtd/nand/raw/nand_onfi.c @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2000 Steven J. Hill (sjhill@realitydiluted.com) + * 2002-2006 Thomas Gleixner (tglx@linutronix.de) + * + * Credits: + * David Woodhouse for adding multichip support + * + * Aleph One Ltd. and Toby Churchill Ltd. for supporting the + * rework for 2K page size chips + * + * This file contains all ONFI helpers. + */ + +#include <linux/slab.h> + +#include "internals.h" + +u16 onfi_crc16(u16 crc, u8 const *p, size_t len) +{ + int i; + while (len--) { + crc ^= *p++ << 8; + for (i = 0; i < 8; i++) + crc = (crc << 1) ^ ((crc & 0x8000) ? 0x8005 : 0); + } + + return crc; +} + +/* Parse the Extended Parameter Page. */ +static int nand_flash_detect_ext_param_page(struct nand_chip *chip, + struct nand_onfi_params *p) +{ + struct onfi_ext_param_page *ep; + struct onfi_ext_section *s; + struct onfi_ext_ecc_info *ecc; + uint8_t *cursor; + int ret; + int len; + int i; + + len = le16_to_cpu(p->ext_param_page_length) * 16; + ep = kmalloc(len, GFP_KERNEL); + if (!ep) + return -ENOMEM; + + /* Send our own NAND_CMD_PARAM. */ + ret = nand_read_param_page_op(chip, 0, NULL, 0); + if (ret) + goto ext_out; + + /* Use the Change Read Column command to skip the ONFI param pages. */ + ret = nand_change_read_column_op(chip, + sizeof(*p) * p->num_of_param_pages, + ep, len, true); + if (ret) + goto ext_out; + + ret = -EINVAL; + if ((onfi_crc16(ONFI_CRC_BASE, ((uint8_t *)ep) + 2, len - 2) + != le16_to_cpu(ep->crc))) { + pr_debug("fail in the CRC.\n"); + goto ext_out; + } + + /* + * Check the signature. + * Do not strictly follow the ONFI spec, maybe changed in future. + */ + if (strncmp(ep->sig, "EPPS", 4)) { + pr_debug("The signature is invalid.\n"); + goto ext_out; + } + + /* find the ECC section. */ + cursor = (uint8_t *)(ep + 1); + for (i = 0; i < ONFI_EXT_SECTION_MAX; i++) { + s = ep->sections + i; + if (s->type == ONFI_SECTION_TYPE_2) + break; + cursor += s->length * 16; + } + if (i == ONFI_EXT_SECTION_MAX) { + pr_debug("We can not find the ECC section.\n"); + goto ext_out; + } + + /* get the info we want. */ + ecc = (struct onfi_ext_ecc_info *)cursor; + + if (!ecc->codeword_size) { + pr_debug("Invalid codeword size\n"); + goto ext_out; + } + + chip->ecc_strength_ds = ecc->ecc_bits; + chip->ecc_step_ds = 1 << ecc->codeword_size; + ret = 0; + +ext_out: + kfree(ep); + return ret; +} + +/* + * Recover data with bit-wise majority + */ +static void nand_bit_wise_majority(const void **srcbufs, + unsigned int nsrcbufs, + void *dstbuf, + unsigned int bufsize) +{ + int i, j, k; + + for (i = 0; i < bufsize; i++) { + u8 val = 0; + + for (j = 0; j < 8; j++) { + unsigned int cnt = 0; + + for (k = 0; k < nsrcbufs; k++) { + const u8 *srcbuf = srcbufs[k]; + + if (srcbuf[i] & BIT(j)) + cnt++; + } + + if (cnt > nsrcbufs / 2) + val |= BIT(j); + } + + ((u8 *)dstbuf)[i] = val; + } +} + +/* + * Check if the NAND chip is ONFI compliant, returns 1 if it is, 0 otherwise. + */ +int nand_onfi_detect(struct nand_chip *chip) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + struct nand_onfi_params *p; + struct onfi_params *onfi; + int onfi_version = 0; + char id[4]; + int i, ret, val; + + /* Try ONFI for unknown chip or LP */ + ret = nand_readid_op(chip, 0x20, id, sizeof(id)); + if (ret || strncmp(id, "ONFI", 4)) + return 0; + + /* ONFI chip: allocate a buffer to hold its parameter page */ + p = kzalloc((sizeof(*p) * 3), GFP_KERNEL); + if (!p) + return -ENOMEM; + + ret = nand_read_param_page_op(chip, 0, NULL, 0); + if (ret) { + ret = 0; + goto free_onfi_param_page; + } + + for (i = 0; i < 3; i++) { + ret = nand_read_data_op(chip, &p[i], sizeof(*p), true); + if (ret) { + ret = 0; + goto free_onfi_param_page; + } + + if (onfi_crc16(ONFI_CRC_BASE, (u8 *)&p[i], 254) == + le16_to_cpu(p->crc)) { + if (i) + memcpy(p, &p[i], sizeof(*p)); + break; + } + } + + if (i == 3) { + const void *srcbufs[3] = {p, p + 1, p + 2}; + + pr_warn("Could not find a valid ONFI parameter page, trying bit-wise majority to recover it\n"); + nand_bit_wise_majority(srcbufs, ARRAY_SIZE(srcbufs), p, + sizeof(*p)); + + if (onfi_crc16(ONFI_CRC_BASE, (u8 *)p, 254) != + le16_to_cpu(p->crc)) { + pr_err("ONFI parameter recovery failed, aborting\n"); + goto free_onfi_param_page; + } + } + + if (chip->manufacturer.desc && chip->manufacturer.desc->ops && + chip->manufacturer.desc->ops->fixup_onfi_param_page) + chip->manufacturer.desc->ops->fixup_onfi_param_page(chip, p); + + /* Check version */ + val = le16_to_cpu(p->revision); + if (val & ONFI_VERSION_2_3) + onfi_version = 23; + else if (val & ONFI_VERSION_2_2) + onfi_version = 22; + else if (val & ONFI_VERSION_2_1) + onfi_version = 21; + else if (val & ONFI_VERSION_2_0) + onfi_version = 20; + else if (val & ONFI_VERSION_1_0) + onfi_version = 10; + + if (!onfi_version) { + pr_info("unsupported ONFI version: %d\n", val); + goto free_onfi_param_page; + } + + sanitize_string(p->manufacturer, sizeof(p->manufacturer)); + sanitize_string(p->model, sizeof(p->model)); + chip->parameters.model = kstrdup(p->model, GFP_KERNEL); + if (!chip->parameters.model) { + ret = -ENOMEM; + goto free_onfi_param_page; + } + + mtd->writesize = le32_to_cpu(p->byte_per_page); + + /* + * pages_per_block and blocks_per_lun may not be a power-of-2 size + * (don't ask me who thought of this...). MTD assumes that these + * dimensions will be power-of-2, so just truncate the remaining area. + */ + mtd->erasesize = 1 << (fls(le32_to_cpu(p->pages_per_block)) - 1); + mtd->erasesize *= mtd->writesize; + + mtd->oobsize = le16_to_cpu(p->spare_bytes_per_page); + + /* See erasesize comment */ + chip->chipsize = 1 << (fls(le32_to_cpu(p->blocks_per_lun)) - 1); + chip->chipsize *= (uint64_t)mtd->erasesize * p->lun_count; + chip->bits_per_cell = p->bits_per_cell; + + chip->max_bb_per_die = le16_to_cpu(p->bb_per_lun); + chip->blocks_per_die = le32_to_cpu(p->blocks_per_lun); + + if (le16_to_cpu(p->features) & ONFI_FEATURE_16_BIT_BUS) + chip->options |= NAND_BUSWIDTH_16; + + if (p->ecc_bits != 0xff) { + chip->ecc_strength_ds = p->ecc_bits; + chip->ecc_step_ds = 512; + } else if (onfi_version >= 21 && + (le16_to_cpu(p->features) & ONFI_FEATURE_EXT_PARAM_PAGE)) { + + /* + * The nand_flash_detect_ext_param_page() uses the + * Change Read Column command which maybe not supported + * by the chip->legacy.cmdfunc. So try to update the + * chip->legacy.cmdfunc now. We do not replace user supplied + * command function. + */ + nand_legacy_adjust_cmdfunc(chip); + + /* The Extended Parameter Page is supported since ONFI 2.1. */ + if (nand_flash_detect_ext_param_page(chip, p)) + pr_warn("Failed to detect ONFI extended param page\n"); + } else { + pr_warn("Could not retrieve ONFI ECC requirements\n"); + } + + /* Save some parameters from the parameter page for future use */ + if (le16_to_cpu(p->opt_cmd) & ONFI_OPT_CMD_SET_GET_FEATURES) { + chip->parameters.supports_set_get_features = true; + bitmap_set(chip->parameters.get_feature_list, + ONFI_FEATURE_ADDR_TIMING_MODE, 1); + bitmap_set(chip->parameters.set_feature_list, + ONFI_FEATURE_ADDR_TIMING_MODE, 1); + } + + onfi = kzalloc(sizeof(*onfi), GFP_KERNEL); + if (!onfi) { + ret = -ENOMEM; + goto free_model; + } + + onfi->version = onfi_version; + onfi->tPROG = le16_to_cpu(p->t_prog); + onfi->tBERS = le16_to_cpu(p->t_bers); + onfi->tR = le16_to_cpu(p->t_r); + onfi->tCCS = le16_to_cpu(p->t_ccs); + onfi->async_timing_mode = le16_to_cpu(p->async_timing_mode); + onfi->vendor_revision = le16_to_cpu(p->vendor_revision); + memcpy(onfi->vendor, p->vendor, sizeof(p->vendor)); + chip->parameters.onfi = onfi; + + /* Identification done, free the full ONFI parameter page and exit */ + kfree(p); + + return 1; + +free_model: + kfree(chip->parameters.model); +free_onfi_param_page: + kfree(p); + + return ret; +} diff --git a/drivers/mtd/nand/raw/nand_samsung.c b/drivers/mtd/nand/raw/nand_samsung.c index ef022f62f74c..e46d4c492ad8 100644 --- a/drivers/mtd/nand/raw/nand_samsung.c +++ b/drivers/mtd/nand/raw/nand_samsung.c @@ -15,7 +15,7 @@ * GNU General Public License for more details. */ -#include <linux/mtd/rawnand.h> +#include "internals.h" static void samsung_nand_decode_id(struct nand_chip *chip) { diff --git a/drivers/mtd/nand/raw/nand_timings.c b/drivers/mtd/nand/raw/nand_timings.c index ebc7b5f76f77..bea3062d71d6 100644 --- a/drivers/mtd/nand/raw/nand_timings.c +++ b/drivers/mtd/nand/raw/nand_timings.c @@ -11,7 +11,8 @@ #include <linux/kernel.h> #include <linux/err.h> #include <linux/export.h> -#include <linux/mtd/rawnand.h> + +#include "internals.h" #define ONFI_DYN_TIMING_MAX U16_MAX @@ -271,20 +272,6 @@ static const struct nand_data_interface onfi_sdr_timings[] = { }; /** - * onfi_async_timing_mode_to_sdr_timings - [NAND Interface] Retrieve NAND - * timings according to the given ONFI timing mode - * @mode: ONFI timing mode - */ -const struct nand_sdr_timings *onfi_async_timing_mode_to_sdr_timings(int mode) -{ - if (mode < 0 || mode >= ARRAY_SIZE(onfi_sdr_timings)) - return ERR_PTR(-EINVAL); - - return &onfi_sdr_timings[mode].timings.sdr; -} -EXPORT_SYMBOL(onfi_async_timing_mode_to_sdr_timings); - -/** * onfi_fill_data_interface - [NAND Interface] Initialize a data interface from * given ONFI mode * @mode: The ONFI timing mode @@ -339,4 +326,3 @@ int onfi_fill_data_interface(struct nand_chip *chip, return 0; } -EXPORT_SYMBOL(onfi_fill_data_interface); diff --git a/drivers/mtd/nand/raw/nand_toshiba.c b/drivers/mtd/nand/raw/nand_toshiba.c index ab43f027cd23..d068163b64b3 100644 --- a/drivers/mtd/nand/raw/nand_toshiba.c +++ b/drivers/mtd/nand/raw/nand_toshiba.c @@ -15,7 +15,88 @@ * GNU General Public License for more details. */ -#include <linux/mtd/rawnand.h> +#include "internals.h" + +/* Bit for detecting BENAND */ +#define TOSHIBA_NAND_ID4_IS_BENAND BIT(7) + +/* Recommended to rewrite for BENAND */ +#define TOSHIBA_NAND_STATUS_REWRITE_RECOMMENDED BIT(3) + +static int toshiba_nand_benand_eccstatus(struct nand_chip *chip) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + int ret; + unsigned int max_bitflips = 0; + u8 status; + + /* Check Status */ + ret = nand_status_op(chip, &status); + if (ret) + return ret; + + if (status & NAND_STATUS_FAIL) { + /* uncorrected */ + mtd->ecc_stats.failed++; + } else if (status & TOSHIBA_NAND_STATUS_REWRITE_RECOMMENDED) { + /* corrected */ + max_bitflips = mtd->bitflip_threshold; + mtd->ecc_stats.corrected += max_bitflips; + } + + return max_bitflips; +} + +static int +toshiba_nand_read_page_benand(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) +{ + int ret; + + ret = nand_read_page_raw(chip, buf, oob_required, page); + if (ret) + return ret; + + return toshiba_nand_benand_eccstatus(chip); +} + +static int +toshiba_nand_read_subpage_benand(struct nand_chip *chip, uint32_t data_offs, + uint32_t readlen, uint8_t *bufpoi, int page) +{ + int ret; + + ret = nand_read_page_op(chip, page, data_offs, + bufpoi + data_offs, readlen); + if (ret) + return ret; + + return toshiba_nand_benand_eccstatus(chip); +} + +static void toshiba_nand_benand_init(struct nand_chip *chip) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + + /* + * On BENAND, the entire OOB region can be used by the MTD user. + * The calculated ECC bytes are stored into other isolated + * area which is not accessible to users. + * This is why chip->ecc.bytes = 0. + */ + chip->ecc.bytes = 0; + chip->ecc.size = 512; + chip->ecc.strength = 8; + chip->ecc.read_page = toshiba_nand_read_page_benand; + chip->ecc.read_subpage = toshiba_nand_read_subpage_benand; + chip->ecc.write_page = nand_write_page_raw; + chip->ecc.read_page_raw = nand_read_page_raw_notsupp; + chip->ecc.write_page_raw = nand_write_page_raw_notsupp; + + chip->options |= NAND_SUBPAGE_READ; + + mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops); +} static void toshiba_nand_decode_id(struct nand_chip *chip) { @@ -68,6 +149,11 @@ static int toshiba_nand_init(struct nand_chip *chip) if (nand_is_slc(chip)) chip->bbt_options |= NAND_BBT_SCAN2NDPAGE; + /* Check that chip is BENAND and ECC mode is on-die */ + if (nand_is_slc(chip) && chip->ecc.mode == NAND_ECC_ON_DIE && + chip->id.data[4] & TOSHIBA_NAND_ID4_IS_BENAND) + toshiba_nand_benand_init(chip); + return 0; } diff --git a/drivers/mtd/nand/raw/nandsim.c b/drivers/mtd/nand/raw/nandsim.c index 71ac034aee9c..c452819f6123 100644 --- a/drivers/mtd/nand/raw/nandsim.c +++ b/drivers/mtd/nand/raw/nandsim.c @@ -656,7 +656,7 @@ static int __init init_nandsim(struct mtd_info *mtd) } /* Force mtd to not do delays */ - chip->chip_delay = 0; + chip->legacy.chip_delay = 0; /* Initialize the NAND flash parameters */ ns->busw = chip->options & NAND_BUSWIDTH_16 ? 16 : 8; @@ -1872,9 +1872,8 @@ static void switch_state(struct nandsim *ns) } } -static u_char ns_nand_read_byte(struct mtd_info *mtd) +static u_char ns_nand_read_byte(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct nandsim *ns = nand_get_controller_data(chip); u_char outb = 0x00; @@ -1934,9 +1933,8 @@ static u_char ns_nand_read_byte(struct mtd_info *mtd) return outb; } -static void ns_nand_write_byte(struct mtd_info *mtd, u_char byte) +static void ns_nand_write_byte(struct nand_chip *chip, u_char byte) { - struct nand_chip *chip = mtd_to_nand(mtd); struct nandsim *ns = nand_get_controller_data(chip); /* Sanity and correctness checks */ @@ -2089,9 +2087,8 @@ static void ns_nand_write_byte(struct mtd_info *mtd, u_char byte) return; } -static void ns_hwcontrol(struct mtd_info *mtd, int cmd, unsigned int bitmask) +static void ns_hwcontrol(struct nand_chip *chip, int cmd, unsigned int bitmask) { - struct nand_chip *chip = mtd_to_nand(mtd); struct nandsim *ns = nand_get_controller_data(chip); ns->lines.cle = bitmask & NAND_CLE ? 1 : 0; @@ -2099,27 +2096,18 @@ static void ns_hwcontrol(struct mtd_info *mtd, int cmd, unsigned int bitmask) ns->lines.ce = bitmask & NAND_NCE ? 1 : 0; if (cmd != NAND_CMD_NONE) - ns_nand_write_byte(mtd, cmd); + ns_nand_write_byte(chip, cmd); } -static int ns_device_ready(struct mtd_info *mtd) +static int ns_device_ready(struct nand_chip *chip) { NS_DBG("device_ready\n"); return 1; } -static uint16_t ns_nand_read_word(struct mtd_info *mtd) +static void ns_nand_write_buf(struct nand_chip *chip, const u_char *buf, + int len) { - struct nand_chip *chip = mtd_to_nand(mtd); - - NS_DBG("read_word\n"); - - return chip->read_byte(mtd) | (chip->read_byte(mtd) << 8); -} - -static void ns_nand_write_buf(struct mtd_info *mtd, const u_char *buf, int len) -{ - struct nand_chip *chip = mtd_to_nand(mtd); struct nandsim *ns = nand_get_controller_data(chip); /* Check that chip is expecting data input */ @@ -2145,9 +2133,8 @@ static void ns_nand_write_buf(struct mtd_info *mtd, const u_char *buf, int len) } } -static void ns_nand_read_buf(struct mtd_info *mtd, u_char *buf, int len) +static void ns_nand_read_buf(struct nand_chip *chip, u_char *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct nandsim *ns = nand_get_controller_data(chip); /* Sanity and correctness checks */ @@ -2169,7 +2156,7 @@ static void ns_nand_read_buf(struct mtd_info *mtd, u_char *buf, int len) int i; for (i = 0; i < len; i++) - buf[i] = mtd_to_nand(mtd)->read_byte(mtd); + buf[i] = chip->legacy.read_byte(chip); return; } @@ -2262,12 +2249,11 @@ static int __init ns_init_module(void) /* * Register simulator's callbacks. */ - chip->cmd_ctrl = ns_hwcontrol; - chip->read_byte = ns_nand_read_byte; - chip->dev_ready = ns_device_ready; - chip->write_buf = ns_nand_write_buf; - chip->read_buf = ns_nand_read_buf; - chip->read_word = ns_nand_read_word; + chip->legacy.cmd_ctrl = ns_hwcontrol; + chip->legacy.read_byte = ns_nand_read_byte; + chip->legacy.dev_ready = ns_device_ready; + chip->legacy.write_buf = ns_nand_write_buf; + chip->legacy.read_buf = ns_nand_read_buf; chip->ecc.mode = NAND_ECC_SOFT; chip->ecc.algo = NAND_ECC_HAMMING; /* The NAND_SKIP_BBTSCAN option is necessary for 'overridesize' */ @@ -2319,7 +2305,7 @@ static int __init ns_init_module(void) goto error; chip->dummy_controller.ops = &ns_controller_ops; - retval = nand_scan(nsmtd, 1); + retval = nand_scan(chip, 1); if (retval) { NS_ERR("Could not scan NAND Simulator device\n"); goto error; @@ -2364,7 +2350,7 @@ static int __init ns_init_module(void) err_exit: free_nandsim(nand); - nand_release(nsmtd); + nand_release(chip); for (i = 0;i < ARRAY_SIZE(nand->partitions); ++i) kfree(nand->partitions[i].name); error: @@ -2386,7 +2372,7 @@ static void __exit ns_cleanup_module(void) int i; free_nandsim(ns); /* Free nandsim private resources */ - nand_release(nsmtd); /* Unregister driver */ + nand_release(chip); /* Unregister driver */ for (i = 0;i < ARRAY_SIZE(ns->partitions); ++i) kfree(ns->partitions[i].name); kfree(mtd_to_nand(nsmtd)); /* Free other structures */ diff --git a/drivers/mtd/nand/raw/ndfc.c b/drivers/mtd/nand/raw/ndfc.c index 540fa1a0ea24..d49a7a17146c 100644 --- a/drivers/mtd/nand/raw/ndfc.c +++ b/drivers/mtd/nand/raw/ndfc.c @@ -44,10 +44,9 @@ struct ndfc_controller { static struct ndfc_controller ndfc_ctrl[NDFC_MAX_CS]; -static void ndfc_select_chip(struct mtd_info *mtd, int chip) +static void ndfc_select_chip(struct nand_chip *nchip, int chip) { uint32_t ccr; - struct nand_chip *nchip = mtd_to_nand(mtd); struct ndfc_controller *ndfc = nand_get_controller_data(nchip); ccr = in_be32(ndfc->ndfcbase + NDFC_CCR); @@ -59,9 +58,8 @@ static void ndfc_select_chip(struct mtd_info *mtd, int chip) out_be32(ndfc->ndfcbase + NDFC_CCR, ccr); } -static void ndfc_hwcontrol(struct mtd_info *mtd, int cmd, unsigned int ctrl) +static void ndfc_hwcontrol(struct nand_chip *chip, int cmd, unsigned int ctrl) { - struct nand_chip *chip = mtd_to_nand(mtd); struct ndfc_controller *ndfc = nand_get_controller_data(chip); if (cmd == NAND_CMD_NONE) @@ -73,18 +71,16 @@ static void ndfc_hwcontrol(struct mtd_info *mtd, int cmd, unsigned int ctrl) writel(cmd & 0xFF, ndfc->ndfcbase + NDFC_ALE); } -static int ndfc_ready(struct mtd_info *mtd) +static int ndfc_ready(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct ndfc_controller *ndfc = nand_get_controller_data(chip); return in_be32(ndfc->ndfcbase + NDFC_STAT) & NDFC_STAT_IS_READY; } -static void ndfc_enable_hwecc(struct mtd_info *mtd, int mode) +static void ndfc_enable_hwecc(struct nand_chip *chip, int mode) { uint32_t ccr; - struct nand_chip *chip = mtd_to_nand(mtd); struct ndfc_controller *ndfc = nand_get_controller_data(chip); ccr = in_be32(ndfc->ndfcbase + NDFC_CCR); @@ -93,10 +89,9 @@ static void ndfc_enable_hwecc(struct mtd_info *mtd, int mode) wmb(); } -static int ndfc_calculate_ecc(struct mtd_info *mtd, +static int ndfc_calculate_ecc(struct nand_chip *chip, const u_char *dat, u_char *ecc_code) { - struct nand_chip *chip = mtd_to_nand(mtd); struct ndfc_controller *ndfc = nand_get_controller_data(chip); uint32_t ecc; uint8_t *p = (uint8_t *)&ecc; @@ -118,9 +113,8 @@ static int ndfc_calculate_ecc(struct mtd_info *mtd, * functions. No further checking, as nand_base will always read/write * page aligned. */ -static void ndfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) +static void ndfc_read_buf(struct nand_chip *chip, uint8_t *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct ndfc_controller *ndfc = nand_get_controller_data(chip); uint32_t *p = (uint32_t *) buf; @@ -128,9 +122,8 @@ static void ndfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) *p++ = in_be32(ndfc->ndfcbase + NDFC_DATA); } -static void ndfc_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len) +static void ndfc_write_buf(struct nand_chip *chip, const uint8_t *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct ndfc_controller *ndfc = nand_get_controller_data(chip); uint32_t *p = (uint32_t *) buf; @@ -149,15 +142,15 @@ static int ndfc_chip_init(struct ndfc_controller *ndfc, struct mtd_info *mtd = nand_to_mtd(chip); int ret; - chip->IO_ADDR_R = ndfc->ndfcbase + NDFC_DATA; - chip->IO_ADDR_W = ndfc->ndfcbase + NDFC_DATA; - chip->cmd_ctrl = ndfc_hwcontrol; - chip->dev_ready = ndfc_ready; + chip->legacy.IO_ADDR_R = ndfc->ndfcbase + NDFC_DATA; + chip->legacy.IO_ADDR_W = ndfc->ndfcbase + NDFC_DATA; + chip->legacy.cmd_ctrl = ndfc_hwcontrol; + chip->legacy.dev_ready = ndfc_ready; chip->select_chip = ndfc_select_chip; - chip->chip_delay = 50; + chip->legacy.chip_delay = 50; chip->controller = &ndfc->ndfc_control; - chip->read_buf = ndfc_read_buf; - chip->write_buf = ndfc_write_buf; + chip->legacy.read_buf = ndfc_read_buf; + chip->legacy.write_buf = ndfc_write_buf; chip->ecc.correct = nand_correct_data; chip->ecc.hwctl = ndfc_enable_hwecc; chip->ecc.calculate = ndfc_calculate_ecc; @@ -174,14 +167,14 @@ static int ndfc_chip_init(struct ndfc_controller *ndfc, return -ENODEV; nand_set_flash_node(chip, flash_np); - mtd->name = kasprintf(GFP_KERNEL, "%s.%s", dev_name(&ndfc->ofdev->dev), - flash_np->name); + mtd->name = kasprintf(GFP_KERNEL, "%s.%pOFn", dev_name(&ndfc->ofdev->dev), + flash_np); if (!mtd->name) { ret = -ENOMEM; goto err; } - ret = nand_scan(mtd, 1); + ret = nand_scan(chip, 1); if (ret) goto err; @@ -258,7 +251,7 @@ static int ndfc_remove(struct platform_device *ofdev) struct ndfc_controller *ndfc = dev_get_drvdata(&ofdev->dev); struct mtd_info *mtd = nand_to_mtd(&ndfc->chip); - nand_release(mtd); + nand_release(&ndfc->chip); kfree(mtd->name); return 0; diff --git a/drivers/mtd/nand/raw/nuc900_nand.c b/drivers/mtd/nand/raw/nuc900_nand.c index af5b32c9a791..38b1994e7ed3 100644 --- a/drivers/mtd/nand/raw/nuc900_nand.c +++ b/drivers/mtd/nand/raw/nuc900_nand.c @@ -79,31 +79,31 @@ static const struct mtd_partition partitions[] = { } }; -static unsigned char nuc900_nand_read_byte(struct mtd_info *mtd) +static unsigned char nuc900_nand_read_byte(struct nand_chip *chip) { unsigned char ret; - struct nuc900_nand *nand = mtd_to_nuc900(mtd); + struct nuc900_nand *nand = mtd_to_nuc900(nand_to_mtd(chip)); ret = (unsigned char)read_data_reg(nand); return ret; } -static void nuc900_nand_read_buf(struct mtd_info *mtd, +static void nuc900_nand_read_buf(struct nand_chip *chip, unsigned char *buf, int len) { int i; - struct nuc900_nand *nand = mtd_to_nuc900(mtd); + struct nuc900_nand *nand = mtd_to_nuc900(nand_to_mtd(chip)); for (i = 0; i < len; i++) buf[i] = (unsigned char)read_data_reg(nand); } -static void nuc900_nand_write_buf(struct mtd_info *mtd, +static void nuc900_nand_write_buf(struct nand_chip *chip, const unsigned char *buf, int len) { int i; - struct nuc900_nand *nand = mtd_to_nuc900(mtd); + struct nuc900_nand *nand = mtd_to_nuc900(nand_to_mtd(chip)); for (i = 0; i < len; i++) write_data_reg(nand, buf[i]); @@ -120,19 +120,20 @@ static int nuc900_check_rb(struct nuc900_nand *nand) return val; } -static int nuc900_nand_devready(struct mtd_info *mtd) +static int nuc900_nand_devready(struct nand_chip *chip) { - struct nuc900_nand *nand = mtd_to_nuc900(mtd); + struct nuc900_nand *nand = mtd_to_nuc900(nand_to_mtd(chip)); int ready; ready = (nuc900_check_rb(nand)) ? 1 : 0; return ready; } -static void nuc900_nand_command_lp(struct mtd_info *mtd, unsigned int command, +static void nuc900_nand_command_lp(struct nand_chip *chip, + unsigned int command, int column, int page_addr) { - register struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(chip); struct nuc900_nand *nand = mtd_to_nuc900(mtd); if (command == NAND_CMD_READOOB) { @@ -174,9 +175,9 @@ static void nuc900_nand_command_lp(struct mtd_info *mtd, unsigned int command, return; case NAND_CMD_RESET: - if (chip->dev_ready) + if (chip->legacy.dev_ready) break; - udelay(chip->chip_delay); + udelay(chip->legacy.chip_delay); write_cmd_reg(nand, NAND_CMD_STATUS); write_cmd_reg(nand, command); @@ -195,8 +196,8 @@ static void nuc900_nand_command_lp(struct mtd_info *mtd, unsigned int command, write_cmd_reg(nand, NAND_CMD_READSTART); default: - if (!chip->dev_ready) { - udelay(chip->chip_delay); + if (!chip->legacy.dev_ready) { + udelay(chip->legacy.chip_delay); return; } } @@ -205,7 +206,7 @@ static void nuc900_nand_command_lp(struct mtd_info *mtd, unsigned int command, * any case on any machine. */ ndelay(100); - while (!chip->dev_ready(mtd)) + while (!chip->legacy.dev_ready(chip)) ; } @@ -253,12 +254,12 @@ static int nuc900_nand_probe(struct platform_device *pdev) return -ENOENT; clk_enable(nuc900_nand->clk); - chip->cmdfunc = nuc900_nand_command_lp; - chip->dev_ready = nuc900_nand_devready; - chip->read_byte = nuc900_nand_read_byte; - chip->write_buf = nuc900_nand_write_buf; - chip->read_buf = nuc900_nand_read_buf; - chip->chip_delay = 50; + chip->legacy.cmdfunc = nuc900_nand_command_lp; + chip->legacy.dev_ready = nuc900_nand_devready; + chip->legacy.read_byte = nuc900_nand_read_byte; + chip->legacy.write_buf = nuc900_nand_write_buf; + chip->legacy.read_buf = nuc900_nand_read_buf; + chip->legacy.chip_delay = 50; chip->options = 0; chip->ecc.mode = NAND_ECC_SOFT; chip->ecc.algo = NAND_ECC_HAMMING; @@ -270,7 +271,7 @@ static int nuc900_nand_probe(struct platform_device *pdev) nuc900_nand_enable(nuc900_nand); - if (nand_scan(mtd, 1)) + if (nand_scan(chip, 1)) return -ENXIO; mtd_device_register(mtd, partitions, ARRAY_SIZE(partitions)); @@ -284,7 +285,7 @@ static int nuc900_nand_remove(struct platform_device *pdev) { struct nuc900_nand *nuc900_nand = platform_get_drvdata(pdev); - nand_release(nand_to_mtd(&nuc900_nand->chip)); + nand_release(&nuc900_nand->chip); clk_disable(nuc900_nand->clk); return 0; diff --git a/drivers/mtd/nand/raw/omap2.c b/drivers/mtd/nand/raw/omap2.c index 4546ac0bed4a..886d05c391ef 100644 --- a/drivers/mtd/nand/raw/omap2.c +++ b/drivers/mtd/nand/raw/omap2.c @@ -240,7 +240,7 @@ static int omap_prefetch_reset(int cs, struct omap_nand_info *info) /** * omap_hwcontrol - hardware specific access to control-lines - * @mtd: MTD device structure + * @chip: NAND chip object * @cmd: command to device * @ctrl: * NAND_NCE: bit 0 -> don't care @@ -249,9 +249,9 @@ static int omap_prefetch_reset(int cs, struct omap_nand_info *info) * * NOTE: boards may use different bits for these!! */ -static void omap_hwcontrol(struct mtd_info *mtd, int cmd, unsigned int ctrl) +static void omap_hwcontrol(struct nand_chip *chip, int cmd, unsigned int ctrl) { - struct omap_nand_info *info = mtd_to_omap(mtd); + struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); if (cmd != NAND_CMD_NONE) { if (ctrl & NAND_CLE) @@ -275,7 +275,7 @@ static void omap_read_buf8(struct mtd_info *mtd, u_char *buf, int len) { struct nand_chip *nand = mtd_to_nand(mtd); - ioread8_rep(nand->IO_ADDR_R, buf, len); + ioread8_rep(nand->legacy.IO_ADDR_R, buf, len); } /** @@ -291,7 +291,7 @@ static void omap_write_buf8(struct mtd_info *mtd, const u_char *buf, int len) bool status; while (len--) { - iowrite8(*p++, info->nand.IO_ADDR_W); + iowrite8(*p++, info->nand.legacy.IO_ADDR_W); /* wait until buffer is available for write */ do { status = info->ops->nand_writebuffer_empty(); @@ -309,7 +309,7 @@ static void omap_read_buf16(struct mtd_info *mtd, u_char *buf, int len) { struct nand_chip *nand = mtd_to_nand(mtd); - ioread16_rep(nand->IO_ADDR_R, buf, len / 2); + ioread16_rep(nand->legacy.IO_ADDR_R, buf, len / 2); } /** @@ -327,7 +327,7 @@ static void omap_write_buf16(struct mtd_info *mtd, const u_char * buf, int len) len >>= 1; while (len--) { - iowrite16(*p++, info->nand.IO_ADDR_W); + iowrite16(*p++, info->nand.legacy.IO_ADDR_W); /* wait until buffer is available for write */ do { status = info->ops->nand_writebuffer_empty(); @@ -337,12 +337,13 @@ static void omap_write_buf16(struct mtd_info *mtd, const u_char * buf, int len) /** * omap_read_buf_pref - read data from NAND controller into buffer - * @mtd: MTD device structure + * @chip: NAND chip object * @buf: buffer to store date * @len: number of bytes to read */ -static void omap_read_buf_pref(struct mtd_info *mtd, u_char *buf, int len) +static void omap_read_buf_pref(struct nand_chip *chip, u_char *buf, int len) { + struct mtd_info *mtd = nand_to_mtd(chip); struct omap_nand_info *info = mtd_to_omap(mtd); uint32_t r_count = 0; int ret = 0; @@ -372,7 +373,7 @@ static void omap_read_buf_pref(struct mtd_info *mtd, u_char *buf, int len) r_count = readl(info->reg.gpmc_prefetch_status); r_count = PREFETCH_STATUS_FIFO_CNT(r_count); r_count = r_count >> 2; - ioread32_rep(info->nand.IO_ADDR_R, p, r_count); + ioread32_rep(info->nand.legacy.IO_ADDR_R, p, r_count); p += r_count; len -= r_count << 2; } while (len); @@ -383,13 +384,14 @@ static void omap_read_buf_pref(struct mtd_info *mtd, u_char *buf, int len) /** * omap_write_buf_pref - write buffer to NAND controller - * @mtd: MTD device structure + * @chip: NAND chip object * @buf: data buffer * @len: number of bytes to write */ -static void omap_write_buf_pref(struct mtd_info *mtd, - const u_char *buf, int len) +static void omap_write_buf_pref(struct nand_chip *chip, const u_char *buf, + int len) { + struct mtd_info *mtd = nand_to_mtd(chip); struct omap_nand_info *info = mtd_to_omap(mtd); uint32_t w_count = 0; int i = 0, ret = 0; @@ -399,7 +401,7 @@ static void omap_write_buf_pref(struct mtd_info *mtd, /* take care of subpage writes */ if (len % 2 != 0) { - writeb(*buf, info->nand.IO_ADDR_W); + writeb(*buf, info->nand.legacy.IO_ADDR_W); p = (u16 *)(buf + 1); len--; } @@ -419,7 +421,7 @@ static void omap_write_buf_pref(struct mtd_info *mtd, w_count = PREFETCH_STATUS_FIFO_CNT(w_count); w_count = w_count >> 1; for (i = 0; (i < w_count) && len; i++, len -= 2) - iowrite16(*p++, info->nand.IO_ADDR_W); + iowrite16(*p++, info->nand.legacy.IO_ADDR_W); } /* wait for data to flushed-out before reset the prefetch */ tim = 0; @@ -528,14 +530,17 @@ out_copy: /** * omap_read_buf_dma_pref - read data from NAND controller into buffer - * @mtd: MTD device structure + * @chip: NAND chip object * @buf: buffer to store date * @len: number of bytes to read */ -static void omap_read_buf_dma_pref(struct mtd_info *mtd, u_char *buf, int len) +static void omap_read_buf_dma_pref(struct nand_chip *chip, u_char *buf, + int len) { + struct mtd_info *mtd = nand_to_mtd(chip); + if (len <= mtd->oobsize) - omap_read_buf_pref(mtd, buf, len); + omap_read_buf_pref(chip, buf, len); else /* start transfer in DMA mode */ omap_nand_dma_transfer(mtd, buf, len, 0x0); @@ -543,18 +548,20 @@ static void omap_read_buf_dma_pref(struct mtd_info *mtd, u_char *buf, int len) /** * omap_write_buf_dma_pref - write buffer to NAND controller - * @mtd: MTD device structure + * @chip: NAND chip object * @buf: data buffer * @len: number of bytes to write */ -static void omap_write_buf_dma_pref(struct mtd_info *mtd, - const u_char *buf, int len) +static void omap_write_buf_dma_pref(struct nand_chip *chip, const u_char *buf, + int len) { + struct mtd_info *mtd = nand_to_mtd(chip); + if (len <= mtd->oobsize) - omap_write_buf_pref(mtd, buf, len); + omap_write_buf_pref(chip, buf, len); else /* start transfer in DMA mode */ - omap_nand_dma_transfer(mtd, (u_char *) buf, len, 0x1); + omap_nand_dma_transfer(mtd, (u_char *)buf, len, 0x1); } /* @@ -578,14 +585,14 @@ static irqreturn_t omap_nand_irq(int this_irq, void *dev) bytes = info->buf_len; else if (!info->buf_len) bytes = 0; - iowrite32_rep(info->nand.IO_ADDR_W, - (u32 *)info->buf, bytes >> 2); + iowrite32_rep(info->nand.legacy.IO_ADDR_W, (u32 *)info->buf, + bytes >> 2); info->buf = info->buf + bytes; info->buf_len -= bytes; } else { - ioread32_rep(info->nand.IO_ADDR_R, - (u32 *)info->buf, bytes >> 2); + ioread32_rep(info->nand.legacy.IO_ADDR_R, (u32 *)info->buf, + bytes >> 2); info->buf = info->buf + bytes; if (this_irq == info->gpmc_irq_count) @@ -605,17 +612,19 @@ done: /* * omap_read_buf_irq_pref - read data from NAND controller into buffer - * @mtd: MTD device structure + * @chip: NAND chip object * @buf: buffer to store date * @len: number of bytes to read */ -static void omap_read_buf_irq_pref(struct mtd_info *mtd, u_char *buf, int len) +static void omap_read_buf_irq_pref(struct nand_chip *chip, u_char *buf, + int len) { + struct mtd_info *mtd = nand_to_mtd(chip); struct omap_nand_info *info = mtd_to_omap(mtd); int ret = 0; if (len <= mtd->oobsize) { - omap_read_buf_pref(mtd, buf, len); + omap_read_buf_pref(chip, buf, len); return; } @@ -651,20 +660,21 @@ out_copy: /* * omap_write_buf_irq_pref - write buffer to NAND controller - * @mtd: MTD device structure + * @chip: NAND chip object * @buf: data buffer * @len: number of bytes to write */ -static void omap_write_buf_irq_pref(struct mtd_info *mtd, - const u_char *buf, int len) +static void omap_write_buf_irq_pref(struct nand_chip *chip, const u_char *buf, + int len) { + struct mtd_info *mtd = nand_to_mtd(chip); struct omap_nand_info *info = mtd_to_omap(mtd); int ret = 0; unsigned long tim, limit; u32 val; if (len <= mtd->oobsize) { - omap_write_buf_pref(mtd, buf, len); + omap_write_buf_pref(chip, buf, len); return; } @@ -857,7 +867,7 @@ static int omap_compare_ecc(u8 *ecc_data1, /* read from NAND memory */ /** * omap_correct_data - Compares the ECC read with HW generated ECC - * @mtd: MTD device structure + * @chip: NAND chip object * @dat: page data * @read_ecc: ecc read from nand flash * @calc_ecc: ecc read from HW ECC registers @@ -869,10 +879,10 @@ static int omap_compare_ecc(u8 *ecc_data1, /* read from NAND memory */ * corrected errors is returned. If uncorrectable errors exist, %-1 is * returned. */ -static int omap_correct_data(struct mtd_info *mtd, u_char *dat, - u_char *read_ecc, u_char *calc_ecc) +static int omap_correct_data(struct nand_chip *chip, u_char *dat, + u_char *read_ecc, u_char *calc_ecc) { - struct omap_nand_info *info = mtd_to_omap(mtd); + struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); int blockCnt = 0, i = 0, ret = 0; int stat = 0; @@ -900,7 +910,7 @@ static int omap_correct_data(struct mtd_info *mtd, u_char *dat, /** * omap_calcuate_ecc - Generate non-inverted ECC bytes. - * @mtd: MTD device structure + * @chip: NAND chip object * @dat: The pointer to data on which ecc is computed * @ecc_code: The ecc_code buffer * @@ -910,10 +920,10 @@ static int omap_correct_data(struct mtd_info *mtd, u_char *dat, * an erased page will produce an ECC mismatch between generated and read * ECC bytes that has to be dealt with separately. */ -static int omap_calculate_ecc(struct mtd_info *mtd, const u_char *dat, - u_char *ecc_code) +static int omap_calculate_ecc(struct nand_chip *chip, const u_char *dat, + u_char *ecc_code) { - struct omap_nand_info *info = mtd_to_omap(mtd); + struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); u32 val; val = readl(info->reg.gpmc_ecc_config); @@ -935,10 +945,9 @@ static int omap_calculate_ecc(struct mtd_info *mtd, const u_char *dat, * @mtd: MTD device structure * @mode: Read/Write mode */ -static void omap_enable_hwecc(struct mtd_info *mtd, int mode) +static void omap_enable_hwecc(struct nand_chip *chip, int mode) { - struct omap_nand_info *info = mtd_to_omap(mtd); - struct nand_chip *chip = mtd_to_nand(mtd); + struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); unsigned int dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0; u32 val; @@ -972,8 +981,7 @@ static void omap_enable_hwecc(struct mtd_info *mtd, int mode) /** * omap_wait - wait until the command is done - * @mtd: MTD device structure - * @chip: NAND Chip structure + * @this: NAND Chip structure * * Wait function is called during Program and erase operations and * the way it is called from MTD layer, we should wait till the NAND @@ -982,10 +990,9 @@ static void omap_enable_hwecc(struct mtd_info *mtd, int mode) * Erase can take up to 400ms and program up to 20ms according to * general NAND and SmartMedia specs */ -static int omap_wait(struct mtd_info *mtd, struct nand_chip *chip) +static int omap_wait(struct nand_chip *this) { - struct nand_chip *this = mtd_to_nand(mtd); - struct omap_nand_info *info = mtd_to_omap(mtd); + struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(this)); unsigned long timeo = jiffies; int status, state = this->state; @@ -1012,9 +1019,9 @@ static int omap_wait(struct mtd_info *mtd, struct nand_chip *chip) * * Returns true if ready and false if busy. */ -static int omap_dev_ready(struct mtd_info *mtd) +static int omap_dev_ready(struct nand_chip *chip) { - struct omap_nand_info *info = mtd_to_omap(mtd); + struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); return gpiod_get_value(info->ready_gpiod); } @@ -1030,13 +1037,13 @@ static int omap_dev_ready(struct mtd_info *mtd) * eccsize0 = 0 (no additional protected byte in spare area) * eccsize1 = 32 (skip 32 nibbles = 16 bytes per sector in spare area) */ -static void __maybe_unused omap_enable_hwecc_bch(struct mtd_info *mtd, int mode) +static void __maybe_unused omap_enable_hwecc_bch(struct nand_chip *chip, + int mode) { unsigned int bch_type; unsigned int dev_width, nsectors; - struct omap_nand_info *info = mtd_to_omap(mtd); + struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); enum omap_ecc ecc_opt = info->ecc_opt; - struct nand_chip *chip = mtd_to_nand(mtd); u32 val, wr_mode; unsigned int ecc_size1, ecc_size0; @@ -1256,7 +1263,7 @@ static int _omap_calculate_ecc_bch(struct mtd_info *mtd, /** * omap_calculate_ecc_bch_sw - ECC generator for sector for SW based correction - * @mtd: MTD device structure + * @chip: NAND chip object * @dat: The pointer to data on which ecc is computed * @ecc_code: The ecc_code buffer * @@ -1264,10 +1271,10 @@ static int _omap_calculate_ecc_bch(struct mtd_info *mtd, * when SW based correction is required as ECC is required for one sector * at a time. */ -static int omap_calculate_ecc_bch_sw(struct mtd_info *mtd, +static int omap_calculate_ecc_bch_sw(struct nand_chip *chip, const u_char *dat, u_char *ecc_calc) { - return _omap_calculate_ecc_bch(mtd, dat, ecc_calc, 0); + return _omap_calculate_ecc_bch(nand_to_mtd(chip), dat, ecc_calc, 0); } /** @@ -1339,7 +1346,7 @@ static int erased_sector_bitflips(u_char *data, u_char *oob, /** * omap_elm_correct_data - corrects page data area in case error reported - * @mtd: MTD device structure + * @chip: NAND chip object * @data: page data * @read_ecc: ecc read from nand flash * @calc_ecc: ecc read from HW ECC registers @@ -1348,10 +1355,10 @@ static int erased_sector_bitflips(u_char *data, u_char *oob, * In case of non-zero ecc vector, first filter out erased-pages, and * then process data via ELM to detect bit-flips. */ -static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data, - u_char *read_ecc, u_char *calc_ecc) +static int omap_elm_correct_data(struct nand_chip *chip, u_char *data, + u_char *read_ecc, u_char *calc_ecc) { - struct omap_nand_info *info = mtd_to_omap(mtd); + struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); struct nand_ecc_ctrl *ecc = &info->nand.ecc; int eccsteps = info->nand.ecc.steps; int i , j, stat = 0; @@ -1512,7 +1519,6 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data, /** * omap_write_page_bch - BCH ecc based write page function for entire page - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: data buffer * @oob_required: must write chip->oob_poi to OOB @@ -1520,19 +1526,20 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data, * * Custom write page method evolved to support multi sector writing in one shot */ -static int omap_write_page_bch(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, int page) +static int omap_write_page_bch(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int ret; uint8_t *ecc_calc = chip->ecc.calc_buf; nand_prog_page_begin_op(chip, page, 0, NULL, 0); /* Enable GPMC ecc engine */ - chip->ecc.hwctl(mtd, NAND_ECC_WRITE); + chip->ecc.hwctl(chip, NAND_ECC_WRITE); /* Write data */ - chip->write_buf(mtd, buf, mtd->writesize); + chip->legacy.write_buf(chip, buf, mtd->writesize); /* Update ecc vector from GPMC result registers */ omap_calculate_ecc_bch_multi(mtd, buf, &ecc_calc[0]); @@ -1543,14 +1550,13 @@ static int omap_write_page_bch(struct mtd_info *mtd, struct nand_chip *chip, return ret; /* Write ecc vector to OOB area */ - chip->write_buf(mtd, chip->oob_poi, mtd->oobsize); + chip->legacy.write_buf(chip, chip->oob_poi, mtd->oobsize); return nand_prog_page_end_op(chip); } /** * omap_write_subpage_bch - BCH hardware ECC based subpage write - * @mtd: mtd info structure * @chip: nand chip info structure * @offset: column address of subpage within the page * @data_len: data length @@ -1560,11 +1566,11 @@ static int omap_write_page_bch(struct mtd_info *mtd, struct nand_chip *chip, * * OMAP optimized subpage write method. */ -static int omap_write_subpage_bch(struct mtd_info *mtd, - struct nand_chip *chip, u32 offset, +static int omap_write_subpage_bch(struct nand_chip *chip, u32 offset, u32 data_len, const u8 *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); u8 *ecc_calc = chip->ecc.calc_buf; int ecc_size = chip->ecc.size; int ecc_bytes = chip->ecc.bytes; @@ -1582,10 +1588,10 @@ static int omap_write_subpage_bch(struct mtd_info *mtd, nand_prog_page_begin_op(chip, page, 0, NULL, 0); /* Enable GPMC ECC engine */ - chip->ecc.hwctl(mtd, NAND_ECC_WRITE); + chip->ecc.hwctl(chip, NAND_ECC_WRITE); /* Write data */ - chip->write_buf(mtd, buf, mtd->writesize); + chip->legacy.write_buf(chip, buf, mtd->writesize); for (step = 0; step < ecc_steps; step++) { /* mask ECC of un-touched subpages by padding 0xFF */ @@ -1610,14 +1616,13 @@ static int omap_write_subpage_bch(struct mtd_info *mtd, return ret; /* write OOB buffer to NAND device */ - chip->write_buf(mtd, chip->oob_poi, mtd->oobsize); + chip->legacy.write_buf(chip, chip->oob_poi, mtd->oobsize); return nand_prog_page_end_op(chip); } /** * omap_read_page_bch - BCH ecc based page read function for entire page - * @mtd: mtd info structure * @chip: nand chip info structure * @buf: buffer to store read data * @oob_required: caller requires OOB data read to chip->oob_poi @@ -1630,9 +1635,10 @@ static int omap_write_subpage_bch(struct mtd_info *mtd, * ecc engine enabled. ecc vector updated after read of OOB data. * For non error pages ecc vector reported as zero. */ -static int omap_read_page_bch(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int omap_read_page_bch(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); uint8_t *ecc_calc = chip->ecc.calc_buf; uint8_t *ecc_code = chip->ecc.code_buf; int stat, ret; @@ -1641,10 +1647,10 @@ static int omap_read_page_bch(struct mtd_info *mtd, struct nand_chip *chip, nand_read_page_op(chip, page, 0, NULL, 0); /* Enable GPMC ecc engine */ - chip->ecc.hwctl(mtd, NAND_ECC_READ); + chip->ecc.hwctl(chip, NAND_ECC_READ); /* Read data */ - chip->read_buf(mtd, buf, mtd->writesize); + chip->legacy.read_buf(chip, buf, mtd->writesize); /* Read oob bytes */ nand_change_read_column_op(chip, @@ -1660,7 +1666,7 @@ static int omap_read_page_bch(struct mtd_info *mtd, struct nand_chip *chip, if (ret) return ret; - stat = chip->ecc.correct(mtd, buf, ecc_code, ecc_calc); + stat = chip->ecc.correct(chip, buf, ecc_code, ecc_calc); if (stat < 0) { mtd->ecc_stats.failed++; @@ -1927,8 +1933,8 @@ static int omap_nand_attach_chip(struct nand_chip *chip) /* Re-populate low-level callbacks based on xfer modes */ switch (info->xfer_type) { case NAND_OMAP_PREFETCH_POLLED: - chip->read_buf = omap_read_buf_pref; - chip->write_buf = omap_write_buf_pref; + chip->legacy.read_buf = omap_read_buf_pref; + chip->legacy.write_buf = omap_write_buf_pref; break; case NAND_OMAP_POLLED: @@ -1960,8 +1966,8 @@ static int omap_nand_attach_chip(struct nand_chip *chip) err); return err; } - chip->read_buf = omap_read_buf_dma_pref; - chip->write_buf = omap_write_buf_dma_pref; + chip->legacy.read_buf = omap_read_buf_dma_pref; + chip->legacy.write_buf = omap_write_buf_dma_pref; } break; @@ -1996,8 +2002,8 @@ static int omap_nand_attach_chip(struct nand_chip *chip) return err; } - chip->read_buf = omap_read_buf_irq_pref; - chip->write_buf = omap_write_buf_irq_pref; + chip->legacy.read_buf = omap_read_buf_irq_pref; + chip->legacy.write_buf = omap_write_buf_irq_pref; break; @@ -2215,16 +2221,16 @@ static int omap_nand_probe(struct platform_device *pdev) } res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - nand_chip->IO_ADDR_R = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(nand_chip->IO_ADDR_R)) - return PTR_ERR(nand_chip->IO_ADDR_R); + nand_chip->legacy.IO_ADDR_R = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(nand_chip->legacy.IO_ADDR_R)) + return PTR_ERR(nand_chip->legacy.IO_ADDR_R); info->phys_base = res->start; nand_chip->controller = &omap_gpmc_controller; - nand_chip->IO_ADDR_W = nand_chip->IO_ADDR_R; - nand_chip->cmd_ctrl = omap_hwcontrol; + nand_chip->legacy.IO_ADDR_W = nand_chip->legacy.IO_ADDR_R; + nand_chip->legacy.cmd_ctrl = omap_hwcontrol; info->ready_gpiod = devm_gpiod_get_optional(&pdev->dev, "rb", GPIOD_IN); @@ -2241,11 +2247,11 @@ static int omap_nand_probe(struct platform_device *pdev) * device and read status register until you get a failure or success */ if (info->ready_gpiod) { - nand_chip->dev_ready = omap_dev_ready; - nand_chip->chip_delay = 0; + nand_chip->legacy.dev_ready = omap_dev_ready; + nand_chip->legacy.chip_delay = 0; } else { - nand_chip->waitfunc = omap_wait; - nand_chip->chip_delay = 50; + nand_chip->legacy.waitfunc = omap_wait; + nand_chip->legacy.chip_delay = 50; } if (info->flash_bbt) @@ -2254,7 +2260,7 @@ static int omap_nand_probe(struct platform_device *pdev) /* scan NAND device connected to chip controller */ nand_chip->options |= info->devsize & NAND_BUSWIDTH_16; - err = nand_scan(mtd, 1); + err = nand_scan(nand_chip, 1); if (err) goto return_error; @@ -2290,7 +2296,7 @@ static int omap_nand_remove(struct platform_device *pdev) } if (info->dma) dma_release_channel(info->dma); - nand_release(mtd); + nand_release(nand_chip); return 0; } diff --git a/drivers/mtd/nand/raw/orion_nand.c b/drivers/mtd/nand/raw/orion_nand.c index 52d435285a3f..d27b39a7223c 100644 --- a/drivers/mtd/nand/raw/orion_nand.c +++ b/drivers/mtd/nand/raw/orion_nand.c @@ -26,9 +26,9 @@ struct orion_nand_info { struct clk *clk; }; -static void orion_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl) +static void orion_nand_cmd_ctrl(struct nand_chip *nc, int cmd, + unsigned int ctrl) { - struct nand_chip *nc = mtd_to_nand(mtd); struct orion_nand_data *board = nand_get_controller_data(nc); u32 offs; @@ -45,13 +45,12 @@ static void orion_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl if (nc->options & NAND_BUSWIDTH_16) offs <<= 1; - writeb(cmd, nc->IO_ADDR_W + offs); + writeb(cmd, nc->legacy.IO_ADDR_W + offs); } -static void orion_nand_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) +static void orion_nand_read_buf(struct nand_chip *chip, uint8_t *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); - void __iomem *io_base = chip->IO_ADDR_R; + void __iomem *io_base = chip->legacy.IO_ADDR_R; #if defined(__LINUX_ARM_ARCH__) && __LINUX_ARM_ARCH__ >= 5 uint64_t *buf64; #endif @@ -137,14 +136,14 @@ static int __init orion_nand_probe(struct platform_device *pdev) nand_set_controller_data(nc, board); nand_set_flash_node(nc, pdev->dev.of_node); - nc->IO_ADDR_R = nc->IO_ADDR_W = io_base; - nc->cmd_ctrl = orion_nand_cmd_ctrl; - nc->read_buf = orion_nand_read_buf; + nc->legacy.IO_ADDR_R = nc->legacy.IO_ADDR_W = io_base; + nc->legacy.cmd_ctrl = orion_nand_cmd_ctrl; + nc->legacy.read_buf = orion_nand_read_buf; nc->ecc.mode = NAND_ECC_SOFT; nc->ecc.algo = NAND_ECC_HAMMING; if (board->chip_delay) - nc->chip_delay = board->chip_delay; + nc->legacy.chip_delay = board->chip_delay; WARN(board->width > 16, "%d bit bus width out of range", @@ -174,14 +173,14 @@ static int __init orion_nand_probe(struct platform_device *pdev) return ret; } - ret = nand_scan(mtd, 1); + ret = nand_scan(nc, 1); if (ret) goto no_dev; mtd->name = "orion_nand"; ret = mtd_device_register(mtd, board->parts, board->nr_parts); if (ret) { - nand_release(mtd); + nand_release(nc); goto no_dev; } @@ -196,9 +195,8 @@ static int orion_nand_remove(struct platform_device *pdev) { struct orion_nand_info *info = platform_get_drvdata(pdev); struct nand_chip *chip = &info->chip; - struct mtd_info *mtd = nand_to_mtd(chip); - nand_release(mtd); + nand_release(chip); clk_disable_unprepare(info->clk); diff --git a/drivers/mtd/nand/raw/oxnas_nand.c b/drivers/mtd/nand/raw/oxnas_nand.c index 01b00bb69c1e..0e52dc29141c 100644 --- a/drivers/mtd/nand/raw/oxnas_nand.c +++ b/drivers/mtd/nand/raw/oxnas_nand.c @@ -38,35 +38,32 @@ struct oxnas_nand_ctrl { struct nand_chip *chips[OXNAS_NAND_MAX_CHIPS]; }; -static uint8_t oxnas_nand_read_byte(struct mtd_info *mtd) +static uint8_t oxnas_nand_read_byte(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct oxnas_nand_ctrl *oxnas = nand_get_controller_data(chip); return readb(oxnas->io_base); } -static void oxnas_nand_read_buf(struct mtd_info *mtd, u8 *buf, int len) +static void oxnas_nand_read_buf(struct nand_chip *chip, u8 *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct oxnas_nand_ctrl *oxnas = nand_get_controller_data(chip); ioread8_rep(oxnas->io_base, buf, len); } -static void oxnas_nand_write_buf(struct mtd_info *mtd, const u8 *buf, int len) +static void oxnas_nand_write_buf(struct nand_chip *chip, const u8 *buf, + int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct oxnas_nand_ctrl *oxnas = nand_get_controller_data(chip); iowrite8_rep(oxnas->io_base, buf, len); } /* Single CS command control */ -static void oxnas_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, +static void oxnas_nand_cmd_ctrl(struct nand_chip *chip, int cmd, unsigned int ctrl) { - struct nand_chip *chip = mtd_to_nand(mtd); struct oxnas_nand_ctrl *oxnas = nand_get_controller_data(chip); if (ctrl & NAND_CLE) @@ -135,20 +132,20 @@ static int oxnas_nand_probe(struct platform_device *pdev) mtd->dev.parent = &pdev->dev; mtd->priv = chip; - chip->cmd_ctrl = oxnas_nand_cmd_ctrl; - chip->read_buf = oxnas_nand_read_buf; - chip->read_byte = oxnas_nand_read_byte; - chip->write_buf = oxnas_nand_write_buf; - chip->chip_delay = 30; + chip->legacy.cmd_ctrl = oxnas_nand_cmd_ctrl; + chip->legacy.read_buf = oxnas_nand_read_buf; + chip->legacy.read_byte = oxnas_nand_read_byte; + chip->legacy.write_buf = oxnas_nand_write_buf; + chip->legacy.chip_delay = 30; /* Scan to find existence of the device */ - err = nand_scan(mtd, 1); + err = nand_scan(chip, 1); if (err) goto err_clk_unprepare; err = mtd_device_register(mtd, NULL, 0); if (err) { - nand_release(mtd); + nand_release(chip); goto err_clk_unprepare; } @@ -176,7 +173,7 @@ static int oxnas_nand_remove(struct platform_device *pdev) struct oxnas_nand_ctrl *oxnas = platform_get_drvdata(pdev); if (oxnas->chips[0]) - nand_release(nand_to_mtd(oxnas->chips[0])); + nand_release(oxnas->chips[0]); clk_disable_unprepare(oxnas->clk); diff --git a/drivers/mtd/nand/raw/pasemi_nand.c b/drivers/mtd/nand/raw/pasemi_nand.c index a47a7e4bd25a..643cd22af009 100644 --- a/drivers/mtd/nand/raw/pasemi_nand.c +++ b/drivers/mtd/nand/raw/pasemi_nand.c @@ -43,49 +43,44 @@ static unsigned int lpcctl; static struct mtd_info *pasemi_nand_mtd; static const char driver_name[] = "pasemi-nand"; -static void pasemi_read_buf(struct mtd_info *mtd, u_char *buf, int len) +static void pasemi_read_buf(struct nand_chip *chip, u_char *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); - while (len > 0x800) { - memcpy_fromio(buf, chip->IO_ADDR_R, 0x800); + memcpy_fromio(buf, chip->legacy.IO_ADDR_R, 0x800); buf += 0x800; len -= 0x800; } - memcpy_fromio(buf, chip->IO_ADDR_R, len); + memcpy_fromio(buf, chip->legacy.IO_ADDR_R, len); } -static void pasemi_write_buf(struct mtd_info *mtd, const u_char *buf, int len) +static void pasemi_write_buf(struct nand_chip *chip, const u_char *buf, + int len) { - struct nand_chip *chip = mtd_to_nand(mtd); - while (len > 0x800) { - memcpy_toio(chip->IO_ADDR_R, buf, 0x800); + memcpy_toio(chip->legacy.IO_ADDR_R, buf, 0x800); buf += 0x800; len -= 0x800; } - memcpy_toio(chip->IO_ADDR_R, buf, len); + memcpy_toio(chip->legacy.IO_ADDR_R, buf, len); } -static void pasemi_hwcontrol(struct mtd_info *mtd, int cmd, +static void pasemi_hwcontrol(struct nand_chip *chip, int cmd, unsigned int ctrl) { - struct nand_chip *chip = mtd_to_nand(mtd); - if (cmd == NAND_CMD_NONE) return; if (ctrl & NAND_CLE) - out_8(chip->IO_ADDR_W + (1 << CLE_PIN_CTL), cmd); + out_8(chip->legacy.IO_ADDR_W + (1 << CLE_PIN_CTL), cmd); else - out_8(chip->IO_ADDR_W + (1 << ALE_PIN_CTL), cmd); + out_8(chip->legacy.IO_ADDR_W + (1 << ALE_PIN_CTL), cmd); /* Push out posted writes */ eieio(); inl(lpcctl); } -int pasemi_device_ready(struct mtd_info *mtd) +int pasemi_device_ready(struct nand_chip *chip) { return !!(inl(lpcctl) & LBICTRL_LPCCTL_NR); } @@ -122,10 +117,10 @@ static int pasemi_nand_probe(struct platform_device *ofdev) /* Link the private data with the MTD structure */ pasemi_nand_mtd->dev.parent = dev; - chip->IO_ADDR_R = of_iomap(np, 0); - chip->IO_ADDR_W = chip->IO_ADDR_R; + chip->legacy.IO_ADDR_R = of_iomap(np, 0); + chip->legacy.IO_ADDR_W = chip->legacy.IO_ADDR_R; - if (!chip->IO_ADDR_R) { + if (!chip->legacy.IO_ADDR_R) { err = -EIO; goto out_mtd; } @@ -144,11 +139,11 @@ static int pasemi_nand_probe(struct platform_device *ofdev) goto out_ior; } - chip->cmd_ctrl = pasemi_hwcontrol; - chip->dev_ready = pasemi_device_ready; - chip->read_buf = pasemi_read_buf; - chip->write_buf = pasemi_write_buf; - chip->chip_delay = 0; + chip->legacy.cmd_ctrl = pasemi_hwcontrol; + chip->legacy.dev_ready = pasemi_device_ready; + chip->legacy.read_buf = pasemi_read_buf; + chip->legacy.write_buf = pasemi_write_buf; + chip->legacy.chip_delay = 0; chip->ecc.mode = NAND_ECC_SOFT; chip->ecc.algo = NAND_ECC_HAMMING; @@ -156,7 +151,7 @@ static int pasemi_nand_probe(struct platform_device *ofdev) chip->bbt_options = NAND_BBT_USE_FLASH; /* Scan to find existence of the device */ - err = nand_scan(pasemi_nand_mtd, 1); + err = nand_scan(chip, 1); if (err) goto out_lpc; @@ -174,7 +169,7 @@ static int pasemi_nand_probe(struct platform_device *ofdev) out_lpc: release_region(lpcctl, 4); out_ior: - iounmap(chip->IO_ADDR_R); + iounmap(chip->legacy.IO_ADDR_R); out_mtd: kfree(chip); out: @@ -191,11 +186,11 @@ static int pasemi_nand_remove(struct platform_device *ofdev) chip = mtd_to_nand(pasemi_nand_mtd); /* Release resources, unregister device */ - nand_release(pasemi_nand_mtd); + nand_release(chip); release_region(lpcctl, 4); - iounmap(chip->IO_ADDR_R); + iounmap(chip->legacy.IO_ADDR_R); /* Free the MTD device structure */ kfree(chip); diff --git a/drivers/mtd/nand/raw/plat_nand.c b/drivers/mtd/nand/raw/plat_nand.c index 222626df4b96..86c536ddaf24 100644 --- a/drivers/mtd/nand/raw/plat_nand.c +++ b/drivers/mtd/nand/raw/plat_nand.c @@ -15,8 +15,7 @@ #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/mtd/mtd.h> -#include <linux/mtd/rawnand.h> -#include <linux/mtd/partitions.h> +#include <linux/mtd/platnand.h> struct plat_nand_data { struct nand_chip chip; @@ -60,14 +59,14 @@ static int plat_nand_probe(struct platform_device *pdev) mtd = nand_to_mtd(&data->chip); mtd->dev.parent = &pdev->dev; - data->chip.IO_ADDR_R = data->io_base; - data->chip.IO_ADDR_W = data->io_base; - data->chip.cmd_ctrl = pdata->ctrl.cmd_ctrl; - data->chip.dev_ready = pdata->ctrl.dev_ready; + data->chip.legacy.IO_ADDR_R = data->io_base; + data->chip.legacy.IO_ADDR_W = data->io_base; + data->chip.legacy.cmd_ctrl = pdata->ctrl.cmd_ctrl; + data->chip.legacy.dev_ready = pdata->ctrl.dev_ready; data->chip.select_chip = pdata->ctrl.select_chip; - data->chip.write_buf = pdata->ctrl.write_buf; - data->chip.read_buf = pdata->ctrl.read_buf; - data->chip.chip_delay = pdata->chip.chip_delay; + data->chip.legacy.write_buf = pdata->ctrl.write_buf; + data->chip.legacy.read_buf = pdata->ctrl.read_buf; + data->chip.legacy.chip_delay = pdata->chip.chip_delay; data->chip.options |= pdata->chip.options; data->chip.bbt_options |= pdata->chip.bbt_options; @@ -84,7 +83,7 @@ static int plat_nand_probe(struct platform_device *pdev) } /* Scan to find existence of the device */ - err = nand_scan(mtd, pdata->chip.nr_chips); + err = nand_scan(&data->chip, pdata->chip.nr_chips); if (err) goto out; @@ -97,7 +96,7 @@ static int plat_nand_probe(struct platform_device *pdev) if (!err) return err; - nand_release(mtd); + nand_release(&data->chip); out: if (pdata->ctrl.remove) pdata->ctrl.remove(pdev); @@ -112,7 +111,7 @@ static int plat_nand_remove(struct platform_device *pdev) struct plat_nand_data *data = platform_get_drvdata(pdev); struct platform_nand_data *pdata = dev_get_platdata(&pdev->dev); - nand_release(nand_to_mtd(&data->chip)); + nand_release(&data->chip); if (pdata->ctrl.remove) pdata->ctrl.remove(pdev); diff --git a/drivers/mtd/nand/raw/qcom_nandc.c b/drivers/mtd/nand/raw/qcom_nandc.c index d1d470bb32e4..ef75dfa62a4f 100644 --- a/drivers/mtd/nand/raw/qcom_nandc.c +++ b/drivers/mtd/nand/raw/qcom_nandc.c @@ -23,7 +23,6 @@ #include <linux/of_device.h> #include <linux/delay.h> #include <linux/dma/qcom_bam_dma.h> -#include <linux/dma-direct.h> /* XXX: drivers shall never use this directly! */ /* NANDc reg offsets */ #define NAND_FLASH_CMD 0x00 @@ -350,7 +349,8 @@ struct nandc_regs { * @data_buffer: our local DMA buffer for page read/writes, * used when we can't use the buffer provided * by upper layers directly - * @buf_size/count/start: markers for chip->read_buf/write_buf functions + * @buf_size/count/start: markers for chip->legacy.read_buf/write_buf + * functions * @reg_read_buf: local buffer for reading back registers via DMA * @reg_read_dma: contains dma address for register read buffer * @reg_read_pos: marker for data read in reg_read_buf @@ -1155,8 +1155,8 @@ static void config_nand_cw_write(struct qcom_nand_controller *nandc) } /* - * the following functions are used within chip->cmdfunc() to perform different - * NAND_CMD_* commands + * the following functions are used within chip->legacy.cmdfunc() to + * perform different NAND_CMD_* commands */ /* sets up descriptors for NAND_CMD_PARAM */ @@ -1436,15 +1436,14 @@ static void post_command(struct qcom_nand_host *host, int command) } /* - * Implements chip->cmdfunc. It's only used for a limited set of commands. - * The rest of the commands wouldn't be called by upper layers. For example, - * NAND_CMD_READOOB would never be called because we have our own versions - * of read_oob ops for nand_ecc_ctrl. + * Implements chip->legacy.cmdfunc. It's only used for a limited set of + * commands. The rest of the commands wouldn't be called by upper layers. + * For example, NAND_CMD_READOOB would never be called because we have our own + * versions of read_oob ops for nand_ecc_ctrl. */ -static void qcom_nandc_command(struct mtd_info *mtd, unsigned int command, +static void qcom_nandc_command(struct nand_chip *chip, unsigned int command, int column, int page_addr) { - struct nand_chip *chip = mtd_to_nand(mtd); struct qcom_nand_host *host = to_qcom_nand_host(chip); struct nand_ecc_ctrl *ecc = &chip->ecc; struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); @@ -1949,8 +1948,8 @@ static int copy_last_cw(struct qcom_nand_host *host, int page) } /* implements ecc->read_page() */ -static int qcom_nandc_read_page(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int qcom_nandc_read_page(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { struct qcom_nand_host *host = to_qcom_nand_host(chip); struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); @@ -1966,10 +1965,10 @@ static int qcom_nandc_read_page(struct mtd_info *mtd, struct nand_chip *chip, } /* implements ecc->read_page_raw() */ -static int qcom_nandc_read_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, uint8_t *buf, +static int qcom_nandc_read_page_raw(struct nand_chip *chip, uint8_t *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct qcom_nand_host *host = to_qcom_nand_host(chip); struct nand_ecc_ctrl *ecc = &chip->ecc; int cw, ret; @@ -1989,8 +1988,7 @@ static int qcom_nandc_read_page_raw(struct mtd_info *mtd, } /* implements ecc->read_oob() */ -static int qcom_nandc_read_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int qcom_nandc_read_oob(struct nand_chip *chip, int page) { struct qcom_nand_host *host = to_qcom_nand_host(chip); struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); @@ -2007,8 +2005,8 @@ static int qcom_nandc_read_oob(struct mtd_info *mtd, struct nand_chip *chip, } /* implements ecc->write_page() */ -static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, int page) +static int qcom_nandc_write_page(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { struct qcom_nand_host *host = to_qcom_nand_host(chip); struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); @@ -2077,10 +2075,11 @@ static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip, } /* implements ecc->write_page_raw() */ -static int qcom_nandc_write_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, const uint8_t *buf, - int oob_required, int page) +static int qcom_nandc_write_page_raw(struct nand_chip *chip, + const uint8_t *buf, int oob_required, + int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct qcom_nand_host *host = to_qcom_nand_host(chip); struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); struct nand_ecc_ctrl *ecc = &chip->ecc; @@ -2155,9 +2154,9 @@ static int qcom_nandc_write_page_raw(struct mtd_info *mtd, * since ECC is calculated for the combined codeword. So update the OOB from * chip->oob_poi, and pad the data area with OxFF before writing. */ -static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int qcom_nandc_write_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct qcom_nand_host *host = to_qcom_nand_host(chip); struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); struct nand_ecc_ctrl *ecc = &chip->ecc; @@ -2197,9 +2196,9 @@ static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip, return nand_prog_page_end_op(chip); } -static int qcom_nandc_block_bad(struct mtd_info *mtd, loff_t ofs) +static int qcom_nandc_block_bad(struct nand_chip *chip, loff_t ofs) { - struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(chip); struct qcom_nand_host *host = to_qcom_nand_host(chip); struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); struct nand_ecc_ctrl *ecc = &chip->ecc; @@ -2235,9 +2234,8 @@ err: return bad; } -static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs) +static int qcom_nandc_block_markbad(struct nand_chip *chip, loff_t ofs) { - struct nand_chip *chip = mtd_to_nand(mtd); struct qcom_nand_host *host = to_qcom_nand_host(chip); struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); struct nand_ecc_ctrl *ecc = &chip->ecc; @@ -2278,14 +2276,13 @@ static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs) } /* - * the three functions below implement chip->read_byte(), chip->read_buf() - * and chip->write_buf() respectively. these aren't used for - * reading/writing page data, they are used for smaller data like reading - * id, status etc + * the three functions below implement chip->legacy.read_byte(), + * chip->legacy.read_buf() and chip->legacy.write_buf() respectively. these + * aren't used for reading/writing page data, they are used for smaller data + * like reading id, status etc */ -static uint8_t qcom_nandc_read_byte(struct mtd_info *mtd) +static uint8_t qcom_nandc_read_byte(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct qcom_nand_host *host = to_qcom_nand_host(chip); struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); u8 *buf = nandc->data_buffer; @@ -2305,9 +2302,8 @@ static uint8_t qcom_nandc_read_byte(struct mtd_info *mtd) return ret; } -static void qcom_nandc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) +static void qcom_nandc_read_buf(struct nand_chip *chip, uint8_t *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start); @@ -2315,10 +2311,9 @@ static void qcom_nandc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) nandc->buf_start += real_len; } -static void qcom_nandc_write_buf(struct mtd_info *mtd, const uint8_t *buf, +static void qcom_nandc_write_buf(struct nand_chip *chip, const uint8_t *buf, int len) { - struct nand_chip *chip = mtd_to_nand(mtd); struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start); @@ -2328,9 +2323,8 @@ static void qcom_nandc_write_buf(struct mtd_info *mtd, const uint8_t *buf, } /* we support only one external chip for now */ -static void qcom_nandc_select_chip(struct mtd_info *mtd, int chipnr) +static void qcom_nandc_select_chip(struct nand_chip *chip, int chipnr) { - struct nand_chip *chip = mtd_to_nand(mtd); struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); if (chipnr <= 0) @@ -2809,13 +2803,13 @@ static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc, mtd->owner = THIS_MODULE; mtd->dev.parent = dev; - chip->cmdfunc = qcom_nandc_command; + chip->legacy.cmdfunc = qcom_nandc_command; chip->select_chip = qcom_nandc_select_chip; - chip->read_byte = qcom_nandc_read_byte; - chip->read_buf = qcom_nandc_read_buf; - chip->write_buf = qcom_nandc_write_buf; - chip->set_features = nand_get_set_features_notsupp; - chip->get_features = nand_get_set_features_notsupp; + chip->legacy.read_byte = qcom_nandc_read_byte; + chip->legacy.read_buf = qcom_nandc_read_buf; + chip->legacy.write_buf = qcom_nandc_write_buf; + chip->legacy.set_features = nand_get_set_features_notsupp; + chip->legacy.get_features = nand_get_set_features_notsupp; /* * the bad block marker is readable only when we read the last codeword @@ -2825,8 +2819,8 @@ static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc, * and block_markbad helpers until we permanently switch to using * MTD_OPS_RAW for all drivers (with the help of badblockbits) */ - chip->block_bad = qcom_nandc_block_bad; - chip->block_markbad = qcom_nandc_block_markbad; + chip->legacy.block_bad = qcom_nandc_block_bad; + chip->legacy.block_markbad = qcom_nandc_block_markbad; chip->controller = &nandc->controller; chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER | @@ -2835,7 +2829,7 @@ static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc, /* set up initial status value */ host->status = NAND_STATUS_READY | NAND_STATUS_WP; - ret = nand_scan(mtd, 1); + ret = nand_scan(chip, 1); if (ret) return ret; @@ -3000,7 +2994,7 @@ static int qcom_nandc_remove(struct platform_device *pdev) struct qcom_nand_host *host; list_for_each_entry(host, &nandc->host_list, node) - nand_release(nand_to_mtd(&host->chip)); + nand_release(&host->chip); qcom_nandc_unalloc(nandc); diff --git a/drivers/mtd/nand/raw/r852.c b/drivers/mtd/nand/raw/r852.c index dcdeb0660e5e..39be65b35ac2 100644 --- a/drivers/mtd/nand/raw/r852.c +++ b/drivers/mtd/nand/raw/r852.c @@ -232,9 +232,9 @@ static void r852_do_dma(struct r852_device *dev, uint8_t *buf, int do_read) /* * Program data lines of the nand chip to send data to it */ -static void r852_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len) +static void r852_write_buf(struct nand_chip *chip, const uint8_t *buf, int len) { - struct r852_device *dev = r852_get_dev(mtd); + struct r852_device *dev = r852_get_dev(nand_to_mtd(chip)); uint32_t reg; /* Don't allow any access to hardware if we suspect card removal */ @@ -266,9 +266,9 @@ static void r852_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len) /* * Read data lines of the nand chip to retrieve data */ -static void r852_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) +static void r852_read_buf(struct nand_chip *chip, uint8_t *buf, int len) { - struct r852_device *dev = r852_get_dev(mtd); + struct r852_device *dev = r852_get_dev(nand_to_mtd(chip)); uint32_t reg; if (dev->card_unstable) { @@ -303,9 +303,9 @@ static void r852_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) /* * Read one byte from nand chip */ -static uint8_t r852_read_byte(struct mtd_info *mtd) +static uint8_t r852_read_byte(struct nand_chip *chip) { - struct r852_device *dev = r852_get_dev(mtd); + struct r852_device *dev = r852_get_dev(nand_to_mtd(chip)); /* Same problem as in r852_read_buf.... */ if (dev->card_unstable) @@ -317,9 +317,9 @@ static uint8_t r852_read_byte(struct mtd_info *mtd) /* * Control several chip lines & send commands */ -static void r852_cmdctl(struct mtd_info *mtd, int dat, unsigned int ctrl) +static void r852_cmdctl(struct nand_chip *chip, int dat, unsigned int ctrl) { - struct r852_device *dev = r852_get_dev(mtd); + struct r852_device *dev = r852_get_dev(nand_to_mtd(chip)); if (dev->card_unstable) return; @@ -362,7 +362,7 @@ static void r852_cmdctl(struct mtd_info *mtd, int dat, unsigned int ctrl) * Wait till card is ready. * based on nand_wait, but returns errors on DMA error */ -static int r852_wait(struct mtd_info *mtd, struct nand_chip *chip) +static int r852_wait(struct nand_chip *chip) { struct r852_device *dev = nand_get_controller_data(chip); @@ -373,7 +373,7 @@ static int r852_wait(struct mtd_info *mtd, struct nand_chip *chip) msecs_to_jiffies(400) : msecs_to_jiffies(20)); while (time_before(jiffies, timeout)) - if (chip->dev_ready(mtd)) + if (chip->legacy.dev_ready(chip)) break; nand_status_op(chip, &status); @@ -390,9 +390,9 @@ static int r852_wait(struct mtd_info *mtd, struct nand_chip *chip) * Check if card is ready */ -static int r852_ready(struct mtd_info *mtd) +static int r852_ready(struct nand_chip *chip) { - struct r852_device *dev = r852_get_dev(mtd); + struct r852_device *dev = r852_get_dev(nand_to_mtd(chip)); return !(r852_read_reg(dev, R852_CARD_STA) & R852_CARD_STA_BUSY); } @@ -401,9 +401,9 @@ static int r852_ready(struct mtd_info *mtd) * Set ECC engine mode */ -static void r852_ecc_hwctl(struct mtd_info *mtd, int mode) +static void r852_ecc_hwctl(struct nand_chip *chip, int mode) { - struct r852_device *dev = r852_get_dev(mtd); + struct r852_device *dev = r852_get_dev(nand_to_mtd(chip)); if (dev->card_unstable) return; @@ -433,10 +433,10 @@ static void r852_ecc_hwctl(struct mtd_info *mtd, int mode) * Calculate ECC, only used for writes */ -static int r852_ecc_calculate(struct mtd_info *mtd, const uint8_t *dat, - uint8_t *ecc_code) +static int r852_ecc_calculate(struct nand_chip *chip, const uint8_t *dat, + uint8_t *ecc_code) { - struct r852_device *dev = r852_get_dev(mtd); + struct r852_device *dev = r852_get_dev(nand_to_mtd(chip)); struct sm_oob *oob = (struct sm_oob *)ecc_code; uint32_t ecc1, ecc2; @@ -465,14 +465,14 @@ static int r852_ecc_calculate(struct mtd_info *mtd, const uint8_t *dat, * Correct the data using ECC, hw did almost everything for us */ -static int r852_ecc_correct(struct mtd_info *mtd, uint8_t *dat, - uint8_t *read_ecc, uint8_t *calc_ecc) +static int r852_ecc_correct(struct nand_chip *chip, uint8_t *dat, + uint8_t *read_ecc, uint8_t *calc_ecc) { uint32_t ecc_reg; uint8_t ecc_status, err_byte; int i, error = 0; - struct r852_device *dev = r852_get_dev(mtd); + struct r852_device *dev = r852_get_dev(nand_to_mtd(chip)); if (dev->card_unstable) return 0; @@ -521,9 +521,10 @@ exit: * This is copy of nand_read_oob_std * nand_read_oob_syndrome assumes we can send column address - we can't */ -static int r852_read_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int r852_read_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + return nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize); } @@ -636,7 +637,7 @@ static int r852_register_nand_device(struct r852_device *dev) { struct mtd_info *mtd = nand_to_mtd(dev->chip); - WARN_ON(dev->card_registred); + WARN_ON(dev->card_registered); mtd->dev.parent = &dev->pci_dev->dev; @@ -653,10 +654,10 @@ static int r852_register_nand_device(struct r852_device *dev) goto error3; } - dev->card_registred = 1; + dev->card_registered = 1; return 0; error3: - nand_release(mtd); + nand_release(dev->chip); error1: /* Force card redetect */ dev->card_detected = 0; @@ -671,13 +672,13 @@ static void r852_unregister_nand_device(struct r852_device *dev) { struct mtd_info *mtd = nand_to_mtd(dev->chip); - if (!dev->card_registred) + if (!dev->card_registered) return; device_remove_file(&mtd->dev, &dev_attr_media_type); - nand_release(mtd); + nand_release(dev->chip); r852_engine_disable(dev); - dev->card_registred = 0; + dev->card_registered = 0; } /* Card state updater */ @@ -691,7 +692,7 @@ static void r852_card_detect_work(struct work_struct *work) dev->card_unstable = 0; /* False alarm */ - if (dev->card_detected == dev->card_registred) + if (dev->card_detected == dev->card_registered) goto exit; /* Read media properties */ @@ -852,14 +853,14 @@ static int r852_probe(struct pci_dev *pci_dev, const struct pci_device_id *id) goto error4; /* commands */ - chip->cmd_ctrl = r852_cmdctl; - chip->waitfunc = r852_wait; - chip->dev_ready = r852_ready; + chip->legacy.cmd_ctrl = r852_cmdctl; + chip->legacy.waitfunc = r852_wait; + chip->legacy.dev_ready = r852_ready; /* I/O */ - chip->read_byte = r852_read_byte; - chip->read_buf = r852_read_buf; - chip->write_buf = r852_write_buf; + chip->legacy.read_byte = r852_read_byte; + chip->legacy.read_buf = r852_read_buf; + chip->legacy.write_buf = r852_write_buf; /* ecc */ chip->ecc.mode = NAND_ECC_HW_SYNDROME; @@ -1025,7 +1026,6 @@ static int r852_suspend(struct device *device) static int r852_resume(struct device *device) { struct r852_device *dev = pci_get_drvdata(to_pci_dev(device)); - struct mtd_info *mtd = nand_to_mtd(dev->chip); r852_disable_irqs(dev); r852_card_update_present(dev); @@ -1033,7 +1033,7 @@ static int r852_resume(struct device *device) /* If card status changed, just do the work */ - if (dev->card_detected != dev->card_registred) { + if (dev->card_detected != dev->card_registered) { dbg("card was %s during low power state", dev->card_detected ? "added" : "removed"); @@ -1043,11 +1043,11 @@ static int r852_resume(struct device *device) } /* Otherwise, initialize the card */ - if (dev->card_registred) { + if (dev->card_registered) { r852_engine_enable(dev); - dev->chip->select_chip(mtd, 0); + dev->chip->select_chip(dev->chip, 0); nand_reset_op(dev->chip); - dev->chip->select_chip(mtd, -1); + dev->chip->select_chip(dev->chip, -1); } /* Program card detection IRQ */ diff --git a/drivers/mtd/nand/raw/r852.h b/drivers/mtd/nand/raw/r852.h index 1eed2fc2fa42..bc67f5bf67e8 100644 --- a/drivers/mtd/nand/raw/r852.h +++ b/drivers/mtd/nand/raw/r852.h @@ -129,7 +129,7 @@ struct r852_device { /* card status area */ struct delayed_work card_detect_work; struct workqueue_struct *card_workqueue; - int card_registred; /* card registered with mtd */ + int card_registered; /* card registered with mtd */ int card_detected; /* card detected in slot */ int card_unstable; /* whenever the card is inserted, is not known yet */ diff --git a/drivers/mtd/nand/raw/s3c2410.c b/drivers/mtd/nand/raw/s3c2410.c index c21e8892394a..d2e42e9d0e8c 100644 --- a/drivers/mtd/nand/raw/s3c2410.c +++ b/drivers/mtd/nand/raw/s3c2410.c @@ -404,7 +404,7 @@ static int s3c2410_nand_inithw(struct s3c2410_nand_info *info) /** * s3c2410_nand_select_chip - select the given nand chip - * @mtd: The MTD instance for this chip. + * @this: NAND chip object. * @chip: The chip number. * * This is called by the MTD layer to either select a given chip for the @@ -415,11 +415,10 @@ static int s3c2410_nand_inithw(struct s3c2410_nand_info *info) * platform specific selection code is called to route nFCE to the specific * chip. */ -static void s3c2410_nand_select_chip(struct mtd_info *mtd, int chip) +static void s3c2410_nand_select_chip(struct nand_chip *this, int chip) { struct s3c2410_nand_info *info; struct s3c2410_nand_mtd *nmtd; - struct nand_chip *this = mtd_to_nand(mtd); unsigned long cur; nmtd = nand_get_controller_data(this); @@ -457,9 +456,10 @@ static void s3c2410_nand_select_chip(struct mtd_info *mtd, int chip) * Issue command and address cycles to the chip */ -static void s3c2410_nand_hwcontrol(struct mtd_info *mtd, int cmd, +static void s3c2410_nand_hwcontrol(struct nand_chip *chip, int cmd, unsigned int ctrl) { + struct mtd_info *mtd = nand_to_mtd(chip); struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); if (cmd == NAND_CMD_NONE) @@ -473,9 +473,10 @@ static void s3c2410_nand_hwcontrol(struct mtd_info *mtd, int cmd, /* command and control functions */ -static void s3c2440_nand_hwcontrol(struct mtd_info *mtd, int cmd, +static void s3c2440_nand_hwcontrol(struct nand_chip *chip, int cmd, unsigned int ctrl) { + struct mtd_info *mtd = nand_to_mtd(chip); struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); if (cmd == NAND_CMD_NONE) @@ -492,29 +493,33 @@ static void s3c2440_nand_hwcontrol(struct mtd_info *mtd, int cmd, * returns 0 if the nand is busy, 1 if it is ready */ -static int s3c2410_nand_devready(struct mtd_info *mtd) +static int s3c2410_nand_devready(struct nand_chip *chip) { + struct mtd_info *mtd = nand_to_mtd(chip); struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); return readb(info->regs + S3C2410_NFSTAT) & S3C2410_NFSTAT_BUSY; } -static int s3c2440_nand_devready(struct mtd_info *mtd) +static int s3c2440_nand_devready(struct nand_chip *chip) { + struct mtd_info *mtd = nand_to_mtd(chip); struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); return readb(info->regs + S3C2440_NFSTAT) & S3C2440_NFSTAT_READY; } -static int s3c2412_nand_devready(struct mtd_info *mtd) +static int s3c2412_nand_devready(struct nand_chip *chip) { + struct mtd_info *mtd = nand_to_mtd(chip); struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); return readb(info->regs + S3C2412_NFSTAT) & S3C2412_NFSTAT_READY; } /* ECC handling functions */ -static int s3c2410_nand_correct_data(struct mtd_info *mtd, u_char *dat, +static int s3c2410_nand_correct_data(struct nand_chip *chip, u_char *dat, u_char *read_ecc, u_char *calc_ecc) { + struct mtd_info *mtd = nand_to_mtd(chip); struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); unsigned int diff0, diff1, diff2; unsigned int bit, byte; @@ -591,38 +596,42 @@ static int s3c2410_nand_correct_data(struct mtd_info *mtd, u_char *dat, * generator block to ECC the data as it passes through] */ -static void s3c2410_nand_enable_hwecc(struct mtd_info *mtd, int mode) +static void s3c2410_nand_enable_hwecc(struct nand_chip *chip, int mode) { - struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); + struct s3c2410_nand_info *info; unsigned long ctrl; + info = s3c2410_nand_mtd_toinfo(nand_to_mtd(chip)); ctrl = readl(info->regs + S3C2410_NFCONF); ctrl |= S3C2410_NFCONF_INITECC; writel(ctrl, info->regs + S3C2410_NFCONF); } -static void s3c2412_nand_enable_hwecc(struct mtd_info *mtd, int mode) +static void s3c2412_nand_enable_hwecc(struct nand_chip *chip, int mode) { - struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); + struct s3c2410_nand_info *info; unsigned long ctrl; + info = s3c2410_nand_mtd_toinfo(nand_to_mtd(chip)); ctrl = readl(info->regs + S3C2440_NFCONT); writel(ctrl | S3C2412_NFCONT_INIT_MAIN_ECC, info->regs + S3C2440_NFCONT); } -static void s3c2440_nand_enable_hwecc(struct mtd_info *mtd, int mode) +static void s3c2440_nand_enable_hwecc(struct nand_chip *chip, int mode) { - struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); + struct s3c2410_nand_info *info; unsigned long ctrl; + info = s3c2410_nand_mtd_toinfo(nand_to_mtd(chip)); ctrl = readl(info->regs + S3C2440_NFCONT); writel(ctrl | S3C2440_NFCONT_INITECC, info->regs + S3C2440_NFCONT); } -static int s3c2410_nand_calculate_ecc(struct mtd_info *mtd, const u_char *dat, - u_char *ecc_code) +static int s3c2410_nand_calculate_ecc(struct nand_chip *chip, + const u_char *dat, u_char *ecc_code) { + struct mtd_info *mtd = nand_to_mtd(chip); struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); ecc_code[0] = readb(info->regs + S3C2410_NFECC + 0); @@ -634,9 +643,10 @@ static int s3c2410_nand_calculate_ecc(struct mtd_info *mtd, const u_char *dat, return 0; } -static int s3c2412_nand_calculate_ecc(struct mtd_info *mtd, const u_char *dat, - u_char *ecc_code) +static int s3c2412_nand_calculate_ecc(struct nand_chip *chip, + const u_char *dat, u_char *ecc_code) { + struct mtd_info *mtd = nand_to_mtd(chip); struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); unsigned long ecc = readl(info->regs + S3C2412_NFMECC0); @@ -649,9 +659,10 @@ static int s3c2412_nand_calculate_ecc(struct mtd_info *mtd, const u_char *dat, return 0; } -static int s3c2440_nand_calculate_ecc(struct mtd_info *mtd, const u_char *dat, - u_char *ecc_code) +static int s3c2440_nand_calculate_ecc(struct nand_chip *chip, + const u_char *dat, u_char *ecc_code) { + struct mtd_info *mtd = nand_to_mtd(chip); struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); unsigned long ecc = readl(info->regs + S3C2440_NFMECC0); @@ -668,14 +679,14 @@ static int s3c2440_nand_calculate_ecc(struct mtd_info *mtd, const u_char *dat, * use read/write block to move the data buffers to/from the controller */ -static void s3c2410_nand_read_buf(struct mtd_info *mtd, u_char *buf, int len) +static void s3c2410_nand_read_buf(struct nand_chip *this, u_char *buf, int len) { - struct nand_chip *this = mtd_to_nand(mtd); - readsb(this->IO_ADDR_R, buf, len); + readsb(this->legacy.IO_ADDR_R, buf, len); } -static void s3c2440_nand_read_buf(struct mtd_info *mtd, u_char *buf, int len) +static void s3c2440_nand_read_buf(struct nand_chip *this, u_char *buf, int len) { + struct mtd_info *mtd = nand_to_mtd(this); struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); readsl(info->regs + S3C2440_NFDATA, buf, len >> 2); @@ -689,16 +700,16 @@ static void s3c2440_nand_read_buf(struct mtd_info *mtd, u_char *buf, int len) } } -static void s3c2410_nand_write_buf(struct mtd_info *mtd, const u_char *buf, +static void s3c2410_nand_write_buf(struct nand_chip *this, const u_char *buf, int len) { - struct nand_chip *this = mtd_to_nand(mtd); - writesb(this->IO_ADDR_W, buf, len); + writesb(this->legacy.IO_ADDR_W, buf, len); } -static void s3c2440_nand_write_buf(struct mtd_info *mtd, const u_char *buf, +static void s3c2440_nand_write_buf(struct nand_chip *this, const u_char *buf, int len) { + struct mtd_info *mtd = nand_to_mtd(this); struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); writesl(info->regs + S3C2440_NFDATA, buf, len >> 2); @@ -781,7 +792,7 @@ static int s3c24xx_nand_remove(struct platform_device *pdev) for (mtdno = 0; mtdno < info->mtd_count; mtdno++, ptr++) { pr_debug("releasing mtd %d (%p)\n", mtdno, ptr); - nand_release(nand_to_mtd(&ptr->chip)); + nand_release(&ptr->chip); } } @@ -809,9 +820,10 @@ static int s3c2410_nand_add_partition(struct s3c2410_nand_info *info, return -ENODEV; } -static int s3c2410_nand_setup_data_interface(struct mtd_info *mtd, int csline, +static int s3c2410_nand_setup_data_interface(struct nand_chip *chip, int csline, const struct nand_data_interface *conf) { + struct mtd_info *mtd = nand_to_mtd(chip); struct s3c2410_nand_info *info = s3c2410_nand_mtd_toinfo(mtd); struct s3c2410_platform_nand *pdata = info->platform; const struct nand_sdr_timings *timings; @@ -852,10 +864,10 @@ static void s3c2410_nand_init_chip(struct s3c2410_nand_info *info, nand_set_flash_node(chip, set->of_node); - chip->write_buf = s3c2410_nand_write_buf; - chip->read_buf = s3c2410_nand_read_buf; + chip->legacy.write_buf = s3c2410_nand_write_buf; + chip->legacy.read_buf = s3c2410_nand_read_buf; chip->select_chip = s3c2410_nand_select_chip; - chip->chip_delay = 50; + chip->legacy.chip_delay = 50; nand_set_controller_data(chip, nmtd); chip->options = set->options; chip->controller = &info->controller; @@ -869,29 +881,29 @@ static void s3c2410_nand_init_chip(struct s3c2410_nand_info *info, switch (info->cpu_type) { case TYPE_S3C2410: - chip->IO_ADDR_W = regs + S3C2410_NFDATA; + chip->legacy.IO_ADDR_W = regs + S3C2410_NFDATA; info->sel_reg = regs + S3C2410_NFCONF; info->sel_bit = S3C2410_NFCONF_nFCE; - chip->cmd_ctrl = s3c2410_nand_hwcontrol; - chip->dev_ready = s3c2410_nand_devready; + chip->legacy.cmd_ctrl = s3c2410_nand_hwcontrol; + chip->legacy.dev_ready = s3c2410_nand_devready; break; case TYPE_S3C2440: - chip->IO_ADDR_W = regs + S3C2440_NFDATA; + chip->legacy.IO_ADDR_W = regs + S3C2440_NFDATA; info->sel_reg = regs + S3C2440_NFCONT; info->sel_bit = S3C2440_NFCONT_nFCE; - chip->cmd_ctrl = s3c2440_nand_hwcontrol; - chip->dev_ready = s3c2440_nand_devready; - chip->read_buf = s3c2440_nand_read_buf; - chip->write_buf = s3c2440_nand_write_buf; + chip->legacy.cmd_ctrl = s3c2440_nand_hwcontrol; + chip->legacy.dev_ready = s3c2440_nand_devready; + chip->legacy.read_buf = s3c2440_nand_read_buf; + chip->legacy.write_buf = s3c2440_nand_write_buf; break; case TYPE_S3C2412: - chip->IO_ADDR_W = regs + S3C2440_NFDATA; + chip->legacy.IO_ADDR_W = regs + S3C2440_NFDATA; info->sel_reg = regs + S3C2440_NFCONT; info->sel_bit = S3C2412_NFCONT_nFCE0; - chip->cmd_ctrl = s3c2440_nand_hwcontrol; - chip->dev_ready = s3c2412_nand_devready; + chip->legacy.cmd_ctrl = s3c2440_nand_hwcontrol; + chip->legacy.dev_ready = s3c2412_nand_devready; if (readl(regs + S3C2410_NFCONF) & S3C2412_NFCONF_NANDBOOT) dev_info(info->device, "System booted from NAND\n"); @@ -899,7 +911,7 @@ static void s3c2410_nand_init_chip(struct s3c2410_nand_info *info, break; } - chip->IO_ADDR_R = chip->IO_ADDR_W; + chip->legacy.IO_ADDR_R = chip->legacy.IO_ADDR_W; nmtd->info = info; nmtd->set = set; @@ -1170,7 +1182,7 @@ static int s3c24xx_nand_probe(struct platform_device *pdev) mtd->dev.parent = &pdev->dev; s3c2410_nand_init_chip(info, nmtd, sets); - err = nand_scan(mtd, sets ? sets->nr_chips : 1); + err = nand_scan(&nmtd->chip, sets ? sets->nr_chips : 1); if (err) goto exit_error; diff --git a/drivers/mtd/nand/raw/sh_flctl.c b/drivers/mtd/nand/raw/sh_flctl.c index bb8866e05ff7..4d20d033de7b 100644 --- a/drivers/mtd/nand/raw/sh_flctl.c +++ b/drivers/mtd/nand/raw/sh_flctl.c @@ -480,7 +480,7 @@ static void read_fiforeg(struct sh_flctl *flctl, int rlen, int offset) /* initiate DMA transfer */ if (flctl->chan_fifo0_rx && rlen >= 32 && - flctl_dma_fifo0_transfer(flctl, buf, rlen, DMA_DEV_TO_MEM) > 0) + flctl_dma_fifo0_transfer(flctl, buf, rlen, DMA_FROM_DEVICE) > 0) goto convert; /* DMA success */ /* do polling transfer */ @@ -539,7 +539,7 @@ static void write_ec_fiforeg(struct sh_flctl *flctl, int rlen, /* initiate DMA transfer */ if (flctl->chan_fifo0_tx && rlen >= 32 && - flctl_dma_fifo0_transfer(flctl, buf, rlen, DMA_MEM_TO_DEV) > 0) + flctl_dma_fifo0_transfer(flctl, buf, rlen, DMA_TO_DEVICE) > 0) return; /* DMA success */ /* do polling transfer */ @@ -611,21 +611,24 @@ static void set_cmd_regs(struct mtd_info *mtd, uint32_t cmd, uint32_t flcmcdr_va writel(flcmcdr_val, FLCMCDR(flctl)); } -static int flctl_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int flctl_read_page_hwecc(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + nand_read_page_op(chip, page, 0, buf, mtd->writesize); if (oob_required) - chip->read_buf(mtd, chip->oob_poi, mtd->oobsize); + chip->legacy.read_buf(chip, chip->oob_poi, mtd->oobsize); return 0; } -static int flctl_write_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, - int page) +static int flctl_write_page_hwecc(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize); - chip->write_buf(mtd, chip->oob_poi, mtd->oobsize); + chip->legacy.write_buf(chip, chip->oob_poi, mtd->oobsize); return nand_prog_page_end_op(chip); } @@ -747,9 +750,10 @@ static void execmd_write_oob(struct mtd_info *mtd) } } -static void flctl_cmdfunc(struct mtd_info *mtd, unsigned int command, +static void flctl_cmdfunc(struct nand_chip *chip, unsigned int command, int column, int page_addr) { + struct mtd_info *mtd = nand_to_mtd(chip); struct sh_flctl *flctl = mtd_to_flctl(mtd); uint32_t read_cmd = 0; @@ -923,9 +927,9 @@ runtime_exit: return; } -static void flctl_select_chip(struct mtd_info *mtd, int chipnr) +static void flctl_select_chip(struct nand_chip *chip, int chipnr) { - struct sh_flctl *flctl = mtd_to_flctl(mtd); + struct sh_flctl *flctl = mtd_to_flctl(nand_to_mtd(chip)); int ret; switch (chipnr) { @@ -967,17 +971,17 @@ static void flctl_select_chip(struct mtd_info *mtd, int chipnr) } } -static void flctl_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len) +static void flctl_write_buf(struct nand_chip *chip, const uint8_t *buf, int len) { - struct sh_flctl *flctl = mtd_to_flctl(mtd); + struct sh_flctl *flctl = mtd_to_flctl(nand_to_mtd(chip)); memcpy(&flctl->done_buff[flctl->index], buf, len); flctl->index += len; } -static uint8_t flctl_read_byte(struct mtd_info *mtd) +static uint8_t flctl_read_byte(struct nand_chip *chip) { - struct sh_flctl *flctl = mtd_to_flctl(mtd); + struct sh_flctl *flctl = mtd_to_flctl(nand_to_mtd(chip)); uint8_t data; data = flctl->done_buff[flctl->index]; @@ -985,18 +989,9 @@ static uint8_t flctl_read_byte(struct mtd_info *mtd) return data; } -static uint16_t flctl_read_word(struct mtd_info *mtd) +static void flctl_read_buf(struct nand_chip *chip, uint8_t *buf, int len) { - struct sh_flctl *flctl = mtd_to_flctl(mtd); - uint16_t *buf = (uint16_t *)&flctl->done_buff[flctl->index]; - - flctl->index += 2; - return *buf; -} - -static void flctl_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) -{ - struct sh_flctl *flctl = mtd_to_flctl(mtd); + struct sh_flctl *flctl = mtd_to_flctl(nand_to_mtd(chip)); memcpy(buf, &flctl->done_buff[flctl->index], len); flctl->index += len; @@ -1183,16 +1178,15 @@ static int flctl_probe(struct platform_device *pdev) /* Set address of hardware control function */ /* 20 us command delay time */ - nand->chip_delay = 20; + nand->legacy.chip_delay = 20; - nand->read_byte = flctl_read_byte; - nand->read_word = flctl_read_word; - nand->write_buf = flctl_write_buf; - nand->read_buf = flctl_read_buf; + nand->legacy.read_byte = flctl_read_byte; + nand->legacy.write_buf = flctl_write_buf; + nand->legacy.read_buf = flctl_read_buf; nand->select_chip = flctl_select_chip; - nand->cmdfunc = flctl_cmdfunc; - nand->set_features = nand_get_set_features_notsupp; - nand->get_features = nand_get_set_features_notsupp; + nand->legacy.cmdfunc = flctl_cmdfunc; + nand->legacy.set_features = nand_get_set_features_notsupp; + nand->legacy.get_features = nand_get_set_features_notsupp; if (pdata->flcmncr_val & SEL_16BIT) nand->options |= NAND_BUSWIDTH_16; @@ -1203,7 +1197,7 @@ static int flctl_probe(struct platform_device *pdev) flctl_setup_dma(flctl); nand->dummy_controller.ops = &flctl_nand_controller_ops; - ret = nand_scan(flctl_mtd, 1); + ret = nand_scan(nand, 1); if (ret) goto err_chip; @@ -1226,7 +1220,7 @@ static int flctl_remove(struct platform_device *pdev) struct sh_flctl *flctl = platform_get_drvdata(pdev); flctl_release_dma(flctl); - nand_release(nand_to_mtd(&flctl->chip)); + nand_release(&flctl->chip); pm_runtime_disable(&pdev->dev); return 0; diff --git a/drivers/mtd/nand/raw/sharpsl.c b/drivers/mtd/nand/raw/sharpsl.c index fc171b17a39b..c82f26c8b58c 100644 --- a/drivers/mtd/nand/raw/sharpsl.c +++ b/drivers/mtd/nand/raw/sharpsl.c @@ -59,11 +59,10 @@ static inline struct sharpsl_nand *mtd_to_sharpsl(struct mtd_info *mtd) * NAND_ALE: bit 2 -> bit 2 * */ -static void sharpsl_nand_hwcontrol(struct mtd_info *mtd, int cmd, +static void sharpsl_nand_hwcontrol(struct nand_chip *chip, int cmd, unsigned int ctrl) { - struct sharpsl_nand *sharpsl = mtd_to_sharpsl(mtd); - struct nand_chip *chip = mtd_to_nand(mtd); + struct sharpsl_nand *sharpsl = mtd_to_sharpsl(nand_to_mtd(chip)); if (ctrl & NAND_CTRL_CHANGE) { unsigned char bits = ctrl & 0x07; @@ -76,24 +75,25 @@ static void sharpsl_nand_hwcontrol(struct mtd_info *mtd, int cmd, } if (cmd != NAND_CMD_NONE) - writeb(cmd, chip->IO_ADDR_W); + writeb(cmd, chip->legacy.IO_ADDR_W); } -static int sharpsl_nand_dev_ready(struct mtd_info *mtd) +static int sharpsl_nand_dev_ready(struct nand_chip *chip) { - struct sharpsl_nand *sharpsl = mtd_to_sharpsl(mtd); + struct sharpsl_nand *sharpsl = mtd_to_sharpsl(nand_to_mtd(chip)); return !((readb(sharpsl->io + FLASHCTL) & FLRYBY) == 0); } -static void sharpsl_nand_enable_hwecc(struct mtd_info *mtd, int mode) +static void sharpsl_nand_enable_hwecc(struct nand_chip *chip, int mode) { - struct sharpsl_nand *sharpsl = mtd_to_sharpsl(mtd); + struct sharpsl_nand *sharpsl = mtd_to_sharpsl(nand_to_mtd(chip)); writeb(0, sharpsl->io + ECCCLRR); } -static int sharpsl_nand_calculate_ecc(struct mtd_info *mtd, const u_char * dat, u_char * ecc_code) +static int sharpsl_nand_calculate_ecc(struct nand_chip *chip, + const u_char * dat, u_char * ecc_code) { - struct sharpsl_nand *sharpsl = mtd_to_sharpsl(mtd); + struct sharpsl_nand *sharpsl = mtd_to_sharpsl(nand_to_mtd(chip)); ecc_code[0] = ~readb(sharpsl->io + ECCLPUB); ecc_code[1] = ~readb(sharpsl->io + ECCLPLB); ecc_code[2] = (~readb(sharpsl->io + ECCCP) << 2) | 0x03; @@ -153,13 +153,13 @@ static int sharpsl_nand_probe(struct platform_device *pdev) writeb(readb(sharpsl->io + FLASHCTL) | FLWP, sharpsl->io + FLASHCTL); /* Set address of NAND IO lines */ - this->IO_ADDR_R = sharpsl->io + FLASHIO; - this->IO_ADDR_W = sharpsl->io + FLASHIO; + this->legacy.IO_ADDR_R = sharpsl->io + FLASHIO; + this->legacy.IO_ADDR_W = sharpsl->io + FLASHIO; /* Set address of hardware control function */ - this->cmd_ctrl = sharpsl_nand_hwcontrol; - this->dev_ready = sharpsl_nand_dev_ready; + this->legacy.cmd_ctrl = sharpsl_nand_hwcontrol; + this->legacy.dev_ready = sharpsl_nand_dev_ready; /* 15 us command delay time */ - this->chip_delay = 15; + this->legacy.chip_delay = 15; /* set eccmode using hardware ECC */ this->ecc.mode = NAND_ECC_HW; this->ecc.size = 256; @@ -171,7 +171,7 @@ static int sharpsl_nand_probe(struct platform_device *pdev) this->ecc.correct = nand_correct_data; /* Scan to find existence of the device */ - err = nand_scan(mtd, 1); + err = nand_scan(this, 1); if (err) goto err_scan; @@ -187,7 +187,7 @@ static int sharpsl_nand_probe(struct platform_device *pdev) return 0; err_add: - nand_release(mtd); + nand_release(this); err_scan: iounmap(sharpsl->io); @@ -205,7 +205,7 @@ static int sharpsl_nand_remove(struct platform_device *pdev) struct sharpsl_nand *sharpsl = platform_get_drvdata(pdev); /* Release resources, unregister device */ - nand_release(nand_to_mtd(&sharpsl->chip)); + nand_release(&sharpsl->chip); iounmap(sharpsl->io); diff --git a/drivers/mtd/nand/raw/sm_common.c b/drivers/mtd/nand/raw/sm_common.c index 73aafe8c3ef3..6f063ef57640 100644 --- a/drivers/mtd/nand/raw/sm_common.c +++ b/drivers/mtd/nand/raw/sm_common.c @@ -99,8 +99,9 @@ static const struct mtd_ooblayout_ops oob_sm_small_ops = { .free = oob_sm_small_ooblayout_free, }; -static int sm_block_markbad(struct mtd_info *mtd, loff_t ofs) +static int sm_block_markbad(struct nand_chip *chip, loff_t ofs) { + struct mtd_info *mtd = nand_to_mtd(chip); struct mtd_oob_ops ops; struct sm_oob oob; int ret; @@ -167,7 +168,7 @@ static int sm_attach_chip(struct nand_chip *chip) /* Bad block marker position */ chip->badblockpos = 0x05; chip->badblockbits = 7; - chip->block_markbad = sm_block_markbad; + chip->legacy.block_markbad = sm_block_markbad; /* ECC layout */ if (mtd->writesize == SM_SECTOR_SIZE) @@ -195,7 +196,7 @@ int sm_register_device(struct mtd_info *mtd, int smartmedia) /* Scan for card properties */ chip->dummy_controller.ops = &sm_controller_ops; flash_ids = smartmedia ? nand_smartmedia_flash_ids : nand_xd_flash_ids; - ret = nand_scan_with_ids(mtd, 1, flash_ids); + ret = nand_scan_with_ids(chip, 1, flash_ids); if (ret) return ret; diff --git a/drivers/mtd/nand/raw/socrates_nand.c b/drivers/mtd/nand/raw/socrates_nand.c index 9824a9923583..8be9a50c7880 100644 --- a/drivers/mtd/nand/raw/socrates_nand.c +++ b/drivers/mtd/nand/raw/socrates_nand.c @@ -34,15 +34,14 @@ struct socrates_nand_host { /** * socrates_nand_write_buf - write buffer to chip - * @mtd: MTD device structure + * @this: NAND chip object * @buf: data buffer * @len: number of bytes to write */ -static void socrates_nand_write_buf(struct mtd_info *mtd, - const uint8_t *buf, int len) +static void socrates_nand_write_buf(struct nand_chip *this, const uint8_t *buf, + int len) { int i; - struct nand_chip *this = mtd_to_nand(mtd); struct socrates_nand_host *host = nand_get_controller_data(this); for (i = 0; i < len; i++) { @@ -54,14 +53,14 @@ static void socrates_nand_write_buf(struct mtd_info *mtd, /** * socrates_nand_read_buf - read chip data into buffer - * @mtd: MTD device structure + * @this: NAND chip object * @buf: buffer to store date * @len: number of bytes to read */ -static void socrates_nand_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) +static void socrates_nand_read_buf(struct nand_chip *this, uint8_t *buf, + int len) { int i; - struct nand_chip *this = mtd_to_nand(mtd); struct socrates_nand_host *host = nand_get_controller_data(this); uint32_t val; @@ -78,31 +77,19 @@ static void socrates_nand_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) * socrates_nand_read_byte - read one byte from the chip * @mtd: MTD device structure */ -static uint8_t socrates_nand_read_byte(struct mtd_info *mtd) +static uint8_t socrates_nand_read_byte(struct nand_chip *this) { uint8_t byte; - socrates_nand_read_buf(mtd, &byte, sizeof(byte)); + socrates_nand_read_buf(this, &byte, sizeof(byte)); return byte; } -/** - * socrates_nand_read_word - read one word from the chip - * @mtd: MTD device structure - */ -static uint16_t socrates_nand_read_word(struct mtd_info *mtd) -{ - uint16_t word; - socrates_nand_read_buf(mtd, (uint8_t *)&word, sizeof(word)); - return word; -} - /* * Hardware specific access to control-lines */ -static void socrates_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, - unsigned int ctrl) +static void socrates_nand_cmd_ctrl(struct nand_chip *nand_chip, int cmd, + unsigned int ctrl) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); struct socrates_nand_host *host = nand_get_controller_data(nand_chip); uint32_t val; @@ -125,9 +112,8 @@ static void socrates_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, /* * Read the Device Ready pin. */ -static int socrates_nand_device_ready(struct mtd_info *mtd) +static int socrates_nand_device_ready(struct nand_chip *nand_chip) { - struct nand_chip *nand_chip = mtd_to_nand(mtd); struct socrates_nand_host *host = nand_get_controller_data(nand_chip); if (in_be32(host->io_base) & FPGA_NAND_BUSY) @@ -166,26 +152,21 @@ static int socrates_nand_probe(struct platform_device *ofdev) mtd->name = "socrates_nand"; mtd->dev.parent = &ofdev->dev; - /*should never be accessed directly */ - nand_chip->IO_ADDR_R = (void *)0xdeadbeef; - nand_chip->IO_ADDR_W = (void *)0xdeadbeef; - - nand_chip->cmd_ctrl = socrates_nand_cmd_ctrl; - nand_chip->read_byte = socrates_nand_read_byte; - nand_chip->read_word = socrates_nand_read_word; - nand_chip->write_buf = socrates_nand_write_buf; - nand_chip->read_buf = socrates_nand_read_buf; - nand_chip->dev_ready = socrates_nand_device_ready; + nand_chip->legacy.cmd_ctrl = socrates_nand_cmd_ctrl; + nand_chip->legacy.read_byte = socrates_nand_read_byte; + nand_chip->legacy.write_buf = socrates_nand_write_buf; + nand_chip->legacy.read_buf = socrates_nand_read_buf; + nand_chip->legacy.dev_ready = socrates_nand_device_ready; nand_chip->ecc.mode = NAND_ECC_SOFT; /* enable ECC */ nand_chip->ecc.algo = NAND_ECC_HAMMING; /* TODO: I have no idea what real delay is. */ - nand_chip->chip_delay = 20; /* 20us command delay time */ + nand_chip->legacy.chip_delay = 20; /* 20us command delay time */ dev_set_drvdata(&ofdev->dev, host); - res = nand_scan(mtd, 1); + res = nand_scan(nand_chip, 1); if (res) goto out; @@ -193,7 +174,7 @@ static int socrates_nand_probe(struct platform_device *ofdev) if (!res) return res; - nand_release(mtd); + nand_release(nand_chip); out: iounmap(host->io_base); @@ -206,9 +187,8 @@ out: static int socrates_nand_remove(struct platform_device *ofdev) { struct socrates_nand_host *host = dev_get_drvdata(&ofdev->dev); - struct mtd_info *mtd = nand_to_mtd(&host->nand_chip); - nand_release(mtd); + nand_release(&host->nand_chip); iounmap(host->io_base); diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c index 1f0b7ee38df5..51b1a548064b 100644 --- a/drivers/mtd/nand/raw/sunxi_nand.c +++ b/drivers/mtd/nand/raw/sunxi_nand.c @@ -400,9 +400,8 @@ static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd, nfc->regs + NFC_REG_CTL); } -static int sunxi_nfc_dev_ready(struct mtd_info *mtd) +static int sunxi_nfc_dev_ready(struct nand_chip *nand) { - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); u32 mask; @@ -420,9 +419,9 @@ static int sunxi_nfc_dev_ready(struct mtd_info *mtd) return !!(readl(nfc->regs + NFC_REG_ST) & mask); } -static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip) +static void sunxi_nfc_select_chip(struct nand_chip *nand, int chip) { - struct nand_chip *nand = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(nand); struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); struct sunxi_nand_chip_sel *sel; @@ -443,9 +442,9 @@ static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip) ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | NFC_PAGE_SHIFT(nand->page_shift); if (sel->rb < 0) { - nand->dev_ready = NULL; + nand->legacy.dev_ready = NULL; } else { - nand->dev_ready = sunxi_nfc_dev_ready; + nand->legacy.dev_ready = sunxi_nfc_dev_ready; ctl |= NFC_RB_SEL(sel->rb); } @@ -464,9 +463,8 @@ static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip) sunxi_nand->selected = chip; } -static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) +static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len) { - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); int ret; @@ -502,10 +500,9 @@ static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) } } -static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf, +static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf, int len) { - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); int ret; @@ -540,19 +537,18 @@ static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf, } } -static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd) +static uint8_t sunxi_nfc_read_byte(struct nand_chip *nand) { uint8_t ret = 0; - sunxi_nfc_read_buf(mtd, &ret, 1); + sunxi_nfc_read_buf(nand, &ret, 1); return ret; } -static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat, +static void sunxi_nfc_cmd_ctrl(struct nand_chip *nand, int dat, unsigned int ctrl) { - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); int ret; @@ -761,7 +757,7 @@ static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd, { sunxi_nfc_randomizer_config(mtd, page, ecc); sunxi_nfc_randomizer_enable(mtd); - sunxi_nfc_write_buf(mtd, buf, len); + sunxi_nfc_write_buf(mtd_to_nand(mtd), buf, len); sunxi_nfc_randomizer_disable(mtd); } @@ -770,7 +766,7 @@ static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf, { sunxi_nfc_randomizer_config(mtd, page, ecc); sunxi_nfc_randomizer_enable(mtd); - sunxi_nfc_read_buf(mtd, buf, len); + sunxi_nfc_read_buf(mtd_to_nand(mtd), buf, len); sunxi_nfc_randomizer_disable(mtd); } @@ -995,7 +991,7 @@ static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd, false); if (!randomize) - sunxi_nfc_read_buf(mtd, oob + offset, len); + sunxi_nfc_read_buf(nand, oob + offset, len); else sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len, false, page); @@ -1189,10 +1185,10 @@ static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd, *cur_off = mtd->oobsize + mtd->writesize; } -static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd, - struct nand_chip *chip, uint8_t *buf, +static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *chip, uint8_t *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct nand_ecc_ctrl *ecc = &chip->ecc; unsigned int max_bitflips = 0; int ret, i, cur_off = 0; @@ -1227,10 +1223,10 @@ static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd, return max_bitflips; } -static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd, - struct nand_chip *chip, u8 *buf, +static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *chip, u8 *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int ret; nand_read_page_op(chip, page, 0, NULL, 0); @@ -1241,14 +1237,14 @@ static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd, return ret; /* Fallback to PIO mode */ - return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page); + return sunxi_nfc_hw_ecc_read_page(chip, buf, oob_required, page); } -static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd, - struct nand_chip *chip, +static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *chip, u32 data_offs, u32 readlen, u8 *bufpoi, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct nand_ecc_ctrl *ecc = &chip->ecc; int ret, i, cur_off = 0; unsigned int max_bitflips = 0; @@ -1278,11 +1274,11 @@ static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd, return max_bitflips; } -static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd, - struct nand_chip *chip, +static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *chip, u32 data_offs, u32 readlen, u8 *buf, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size); int ret; @@ -1293,15 +1289,15 @@ static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd, return ret; /* Fallback to PIO mode */ - return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen, + return sunxi_nfc_hw_ecc_read_subpage(chip, data_offs, readlen, buf, page); } -static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd, - struct nand_chip *chip, +static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *chip, const uint8_t *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct nand_ecc_ctrl *ecc = &chip->ecc; int ret, i, cur_off = 0; @@ -1331,12 +1327,12 @@ static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd, return nand_prog_page_end_op(chip); } -static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd, - struct nand_chip *chip, +static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *chip, u32 data_offs, u32 data_len, const u8 *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct nand_ecc_ctrl *ecc = &chip->ecc; int ret, i, cur_off = 0; @@ -1363,12 +1359,12 @@ static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd, return nand_prog_page_end_op(chip); } -static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd, - struct nand_chip *chip, +static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *chip, const u8 *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); struct nand_ecc_ctrl *ecc = &nand->ecc; @@ -1425,28 +1421,25 @@ static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd, return nand_prog_page_end_op(chip); pio_fallback: - return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page); + return sunxi_nfc_hw_ecc_write_page(chip, buf, oob_required, page); } -static int sunxi_nfc_hw_ecc_read_oob(struct mtd_info *mtd, - struct nand_chip *chip, - int page) +static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *chip, int page) { chip->pagebuf = -1; - return chip->ecc.read_page(mtd, chip, chip->data_buf, 1, page); + return chip->ecc.read_page(chip, chip->data_buf, 1, page); } -static int sunxi_nfc_hw_ecc_write_oob(struct mtd_info *mtd, - struct nand_chip *chip, - int page) +static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); int ret; chip->pagebuf = -1; memset(chip->data_buf, 0xff, mtd->writesize); - ret = chip->ecc.write_page(mtd, chip, chip->data_buf, 1, page); + ret = chip->ecc.write_page(chip, chip->data_buf, 1, page); if (ret) return ret; @@ -1475,10 +1468,9 @@ static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration, #define sunxi_nand_lookup_timing(l, p, c) \ _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c) -static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd, int csline, +static int sunxi_nfc_setup_data_interface(struct nand_chip *nand, int csline, const struct nand_data_interface *conf) { - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nand_chip *chip = to_sunxi_nand(nand); struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller); const struct nand_sdr_timings *timings; @@ -1920,7 +1912,7 @@ static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc, nand = &chip->nand; /* Default tR value specified in the ONFI spec (chapter 4.15.1) */ - nand->chip_delay = 200; + nand->legacy.chip_delay = 200; nand->controller = &nfc->controller; nand->controller->ops = &sunxi_nand_controller_ops; @@ -1931,23 +1923,23 @@ static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc, nand->ecc.mode = NAND_ECC_HW; nand_set_flash_node(nand, np); nand->select_chip = sunxi_nfc_select_chip; - nand->cmd_ctrl = sunxi_nfc_cmd_ctrl; - nand->read_buf = sunxi_nfc_read_buf; - nand->write_buf = sunxi_nfc_write_buf; - nand->read_byte = sunxi_nfc_read_byte; + nand->legacy.cmd_ctrl = sunxi_nfc_cmd_ctrl; + nand->legacy.read_buf = sunxi_nfc_read_buf; + nand->legacy.write_buf = sunxi_nfc_write_buf; + nand->legacy.read_byte = sunxi_nfc_read_byte; nand->setup_data_interface = sunxi_nfc_setup_data_interface; mtd = nand_to_mtd(nand); mtd->dev.parent = dev; - ret = nand_scan(mtd, nsels); + ret = nand_scan(nand, nsels); if (ret) return ret; ret = mtd_device_register(mtd, NULL, 0); if (ret) { dev_err(dev, "failed to register mtd device: %d\n", ret); - nand_release(mtd); + nand_release(nand); return ret; } @@ -1986,7 +1978,7 @@ static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc) while (!list_empty(&nfc->chips)) { chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip, node); - nand_release(nand_to_mtd(&chip->nand)); + nand_release(&chip->nand); sunxi_nand_ecc_cleanup(&chip->nand.ecc); list_del(&chip->node); } diff --git a/drivers/mtd/nand/raw/tango_nand.c b/drivers/mtd/nand/raw/tango_nand.c index 72698691727d..8818f893f300 100644 --- a/drivers/mtd/nand/raw/tango_nand.c +++ b/drivers/mtd/nand/raw/tango_nand.c @@ -116,9 +116,9 @@ struct tango_chip { #define TIMING(t0, t1, t2, t3) ((t0) << 24 | (t1) << 16 | (t2) << 8 | (t3)) -static void tango_cmd_ctrl(struct mtd_info *mtd, int dat, unsigned int ctrl) +static void tango_cmd_ctrl(struct nand_chip *chip, int dat, unsigned int ctrl) { - struct tango_chip *tchip = to_tango_chip(mtd_to_nand(mtd)); + struct tango_chip *tchip = to_tango_chip(chip); if (ctrl & NAND_CLE) writeb_relaxed(dat, tchip->base + PBUS_CMD); @@ -127,38 +127,36 @@ static void tango_cmd_ctrl(struct mtd_info *mtd, int dat, unsigned int ctrl) writeb_relaxed(dat, tchip->base + PBUS_ADDR); } -static int tango_dev_ready(struct mtd_info *mtd) +static int tango_dev_ready(struct nand_chip *chip) { - struct nand_chip *chip = mtd_to_nand(mtd); struct tango_nfc *nfc = to_tango_nfc(chip->controller); return readl_relaxed(nfc->pbus_base + PBUS_CS_CTRL) & PBUS_IORDY; } -static u8 tango_read_byte(struct mtd_info *mtd) +static u8 tango_read_byte(struct nand_chip *chip) { - struct tango_chip *tchip = to_tango_chip(mtd_to_nand(mtd)); + struct tango_chip *tchip = to_tango_chip(chip); return readb_relaxed(tchip->base + PBUS_DATA); } -static void tango_read_buf(struct mtd_info *mtd, u8 *buf, int len) +static void tango_read_buf(struct nand_chip *chip, u8 *buf, int len) { - struct tango_chip *tchip = to_tango_chip(mtd_to_nand(mtd)); + struct tango_chip *tchip = to_tango_chip(chip); ioread8_rep(tchip->base + PBUS_DATA, buf, len); } -static void tango_write_buf(struct mtd_info *mtd, const u8 *buf, int len) +static void tango_write_buf(struct nand_chip *chip, const u8 *buf, int len) { - struct tango_chip *tchip = to_tango_chip(mtd_to_nand(mtd)); + struct tango_chip *tchip = to_tango_chip(chip); iowrite8_rep(tchip->base + PBUS_DATA, buf, len); } -static void tango_select_chip(struct mtd_info *mtd, int idx) +static void tango_select_chip(struct nand_chip *chip, int idx) { - struct nand_chip *chip = mtd_to_nand(mtd); struct tango_nfc *nfc = to_tango_nfc(chip->controller); struct tango_chip *tchip = to_tango_chip(chip); @@ -277,14 +275,15 @@ dma_unmap: return err; } -static int tango_read_page(struct mtd_info *mtd, struct nand_chip *chip, - u8 *buf, int oob_required, int page) +static int tango_read_page(struct nand_chip *chip, u8 *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct tango_nfc *nfc = to_tango_nfc(chip->controller); int err, res, len = mtd->writesize; if (oob_required) - chip->ecc.read_oob(mtd, chip, page); + chip->ecc.read_oob(chip, page); err = do_dma(nfc, DMA_FROM_DEVICE, NFC_READ, buf, len, page); if (err) @@ -292,16 +291,17 @@ static int tango_read_page(struct mtd_info *mtd, struct nand_chip *chip, res = decode_error_report(chip); if (res < 0) { - chip->ecc.read_oob_raw(mtd, chip, page); + chip->ecc.read_oob_raw(chip, page); res = check_erased_page(chip, buf); } return res; } -static int tango_write_page(struct mtd_info *mtd, struct nand_chip *chip, - const u8 *buf, int oob_required, int page) +static int tango_write_page(struct nand_chip *chip, const u8 *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct tango_nfc *nfc = to_tango_nfc(chip->controller); int err, status, len = mtd->writesize; @@ -314,7 +314,7 @@ static int tango_write_page(struct mtd_info *mtd, struct nand_chip *chip, if (err) return err; - status = chip->waitfunc(mtd, chip); + status = chip->legacy.waitfunc(chip); if (status & NAND_STATUS_FAIL) return -EIO; @@ -323,30 +323,26 @@ static int tango_write_page(struct mtd_info *mtd, struct nand_chip *chip, static void aux_read(struct nand_chip *chip, u8 **buf, int len, int *pos) { - struct mtd_info *mtd = nand_to_mtd(chip); - *pos += len; if (!*buf) { /* skip over "len" bytes */ nand_change_read_column_op(chip, *pos, NULL, 0, false); } else { - tango_read_buf(mtd, *buf, len); + tango_read_buf(chip, *buf, len); *buf += len; } } static void aux_write(struct nand_chip *chip, const u8 **buf, int len, int *pos) { - struct mtd_info *mtd = nand_to_mtd(chip); - *pos += len; if (!*buf) { /* skip over "len" bytes */ nand_change_write_column_op(chip, *pos, NULL, 0, false); } else { - tango_write_buf(mtd, *buf, len); + tango_write_buf(chip, *buf, len); *buf += len; } } @@ -424,32 +420,30 @@ static void raw_write(struct nand_chip *chip, const u8 *buf, const u8 *oob) aux_write(chip, &oob, ecc_size, &pos); } -static int tango_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip, - u8 *buf, int oob_required, int page) +static int tango_read_page_raw(struct nand_chip *chip, u8 *buf, + int oob_required, int page) { nand_read_page_op(chip, page, 0, NULL, 0); raw_read(chip, buf, chip->oob_poi); return 0; } -static int tango_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip, - const u8 *buf, int oob_required, int page) +static int tango_write_page_raw(struct nand_chip *chip, const u8 *buf, + int oob_required, int page) { nand_prog_page_begin_op(chip, page, 0, NULL, 0); raw_write(chip, buf, chip->oob_poi); return nand_prog_page_end_op(chip); } -static int tango_read_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int tango_read_oob(struct nand_chip *chip, int page) { nand_read_page_op(chip, page, 0, NULL, 0); raw_read(chip, NULL, chip->oob_poi); return 0; } -static int tango_write_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int tango_write_oob(struct nand_chip *chip, int page) { nand_prog_page_begin_op(chip, page, 0, NULL, 0); raw_write(chip, NULL, chip->oob_poi); @@ -485,11 +479,10 @@ static u32 to_ticks(int kHz, int ps) return DIV_ROUND_UP_ULL((u64)kHz * ps, NSEC_PER_SEC); } -static int tango_set_timings(struct mtd_info *mtd, int csline, +static int tango_set_timings(struct nand_chip *chip, int csline, const struct nand_data_interface *conf) { const struct nand_sdr_timings *sdr = nand_get_sdr_timings(conf); - struct nand_chip *chip = mtd_to_nand(mtd); struct tango_nfc *nfc = to_tango_nfc(chip->controller); struct tango_chip *tchip = to_tango_chip(chip); u32 Trdy, Textw, Twc, Twpw, Tacc, Thold, Trpw, Textr; @@ -571,12 +564,12 @@ static int chip_init(struct device *dev, struct device_node *np) ecc = &chip->ecc; mtd = nand_to_mtd(chip); - chip->read_byte = tango_read_byte; - chip->write_buf = tango_write_buf; - chip->read_buf = tango_read_buf; + chip->legacy.read_byte = tango_read_byte; + chip->legacy.write_buf = tango_write_buf; + chip->legacy.read_buf = tango_read_buf; chip->select_chip = tango_select_chip; - chip->cmd_ctrl = tango_cmd_ctrl; - chip->dev_ready = tango_dev_ready; + chip->legacy.cmd_ctrl = tango_cmd_ctrl; + chip->legacy.dev_ready = tango_dev_ready; chip->setup_data_interface = tango_set_timings; chip->options = NAND_USE_BOUNCE_BUFFER | NAND_NO_SUBPAGE_WRITE | @@ -588,7 +581,7 @@ static int chip_init(struct device *dev, struct device_node *np) mtd_set_ooblayout(mtd, &tango_nand_ooblayout_ops); mtd->dev.parent = dev; - err = nand_scan(mtd, 1); + err = nand_scan(chip, 1); if (err) return err; @@ -617,7 +610,7 @@ static int tango_nand_remove(struct platform_device *pdev) for (cs = 0; cs < MAX_CS; ++cs) { if (nfc->chips[cs]) - nand_release(nand_to_mtd(&nfc->chips[cs]->nand_chip)); + nand_release(&nfc->chips[cs]->nand_chip); } return 0; diff --git a/drivers/mtd/nand/raw/tegra_nand.c b/drivers/mtd/nand/raw/tegra_nand.c index 79da1efc88d1..9767e29d74e2 100644 --- a/drivers/mtd/nand/raw/tegra_nand.c +++ b/drivers/mtd/nand/raw/tegra_nand.c @@ -462,9 +462,8 @@ static int tegra_nand_exec_op(struct nand_chip *chip, check_only); } -static void tegra_nand_select_chip(struct mtd_info *mtd, int die_nr) +static void tegra_nand_select_chip(struct nand_chip *chip, int die_nr) { - struct nand_chip *chip = mtd_to_nand(mtd); struct tegra_nand_chip *nand = to_tegra_chip(chip); struct tegra_nand_controller *ctrl = to_tegra_ctrl(chip->controller); @@ -615,44 +614,46 @@ err_unmap_dma_page: return ret; } -static int tegra_nand_read_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, u8 *buf, +static int tegra_nand_read_page_raw(struct nand_chip *chip, u8 *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); void *oob_buf = oob_required ? chip->oob_poi : NULL; return tegra_nand_page_xfer(mtd, chip, buf, oob_buf, mtd->oobsize, page, true); } -static int tegra_nand_write_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, const u8 *buf, +static int tegra_nand_write_page_raw(struct nand_chip *chip, const u8 *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); void *oob_buf = oob_required ? chip->oob_poi : NULL; return tegra_nand_page_xfer(mtd, chip, (void *)buf, oob_buf, mtd->oobsize, page, false); } -static int tegra_nand_read_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int tegra_nand_read_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + return tegra_nand_page_xfer(mtd, chip, NULL, chip->oob_poi, mtd->oobsize, page, true); } -static int tegra_nand_write_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int tegra_nand_write_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); + return tegra_nand_page_xfer(mtd, chip, NULL, chip->oob_poi, mtd->oobsize, page, false); } -static int tegra_nand_read_page_hwecc(struct mtd_info *mtd, - struct nand_chip *chip, u8 *buf, +static int tegra_nand_read_page_hwecc(struct nand_chip *chip, u8 *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct tegra_nand_controller *ctrl = to_tegra_ctrl(chip->controller); struct tegra_nand_chip *nand = to_tegra_chip(chip); void *oob_buf = oob_required ? chip->oob_poi : NULL; @@ -716,7 +717,7 @@ static int tegra_nand_read_page_hwecc(struct mtd_info *mtd, * erased or if error correction just failed for all sub- * pages. */ - ret = tegra_nand_read_oob(mtd, chip, page); + ret = tegra_nand_read_oob(chip, page); if (ret < 0) return ret; @@ -759,10 +760,10 @@ static int tegra_nand_read_page_hwecc(struct mtd_info *mtd, } } -static int tegra_nand_write_page_hwecc(struct mtd_info *mtd, - struct nand_chip *chip, const u8 *buf, +static int tegra_nand_write_page_hwecc(struct nand_chip *chip, const u8 *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct tegra_nand_controller *ctrl = to_tegra_ctrl(chip->controller); void *oob_buf = oob_required ? chip->oob_poi : NULL; int ret; @@ -813,10 +814,9 @@ static void tegra_nand_setup_timing(struct tegra_nand_controller *ctrl, writel_relaxed(reg, ctrl->regs + TIMING_2); } -static int tegra_nand_setup_data_interface(struct mtd_info *mtd, int csline, +static int tegra_nand_setup_data_interface(struct nand_chip *chip, int csline, const struct nand_data_interface *conf) { - struct nand_chip *chip = mtd_to_nand(mtd); struct tegra_nand_controller *ctrl = to_tegra_ctrl(chip->controller); const struct nand_sdr_timings *timings; @@ -1119,7 +1119,7 @@ static int tegra_nand_chips_init(struct device *dev, chip->select_chip = tegra_nand_select_chip; chip->setup_data_interface = tegra_nand_setup_data_interface; - ret = nand_scan(mtd, 1); + ret = nand_scan(chip, 1); if (ret) return ret; diff --git a/drivers/mtd/nand/raw/tmio_nand.c b/drivers/mtd/nand/raw/tmio_nand.c index dcaa924502de..f3b59e649b7d 100644 --- a/drivers/mtd/nand/raw/tmio_nand.c +++ b/drivers/mtd/nand/raw/tmio_nand.c @@ -126,11 +126,10 @@ static inline struct tmio_nand *mtd_to_tmio(struct mtd_info *mtd) /*--------------------------------------------------------------------------*/ -static void tmio_nand_hwcontrol(struct mtd_info *mtd, int cmd, - unsigned int ctrl) +static void tmio_nand_hwcontrol(struct nand_chip *chip, int cmd, + unsigned int ctrl) { - struct tmio_nand *tmio = mtd_to_tmio(mtd); - struct nand_chip *chip = mtd_to_nand(mtd); + struct tmio_nand *tmio = mtd_to_tmio(nand_to_mtd(chip)); if (ctrl & NAND_CTRL_CHANGE) { u8 mode; @@ -156,12 +155,12 @@ static void tmio_nand_hwcontrol(struct mtd_info *mtd, int cmd, } if (cmd != NAND_CMD_NONE) - tmio_iowrite8(cmd, chip->IO_ADDR_W); + tmio_iowrite8(cmd, chip->legacy.IO_ADDR_W); } -static int tmio_nand_dev_ready(struct mtd_info *mtd) +static int tmio_nand_dev_ready(struct nand_chip *chip) { - struct tmio_nand *tmio = mtd_to_tmio(mtd); + struct tmio_nand *tmio = mtd_to_tmio(nand_to_mtd(chip)); return !(tmio_ioread8(tmio->fcr + FCR_STATUS) & FCR_STATUS_BUSY); } @@ -187,10 +186,9 @@ static irqreturn_t tmio_irq(int irq, void *__tmio) *erase and write, we enable it to wake us up. The irq handler *disables the interrupt. */ -static int -tmio_nand_wait(struct mtd_info *mtd, struct nand_chip *nand_chip) +static int tmio_nand_wait(struct nand_chip *nand_chip) { - struct tmio_nand *tmio = mtd_to_tmio(mtd); + struct tmio_nand *tmio = mtd_to_tmio(nand_to_mtd(nand_chip)); long timeout; u8 status; @@ -199,10 +197,10 @@ tmio_nand_wait(struct mtd_info *mtd, struct nand_chip *nand_chip) tmio_iowrite8(0x81, tmio->fcr + FCR_IMR); timeout = wait_event_timeout(nand_chip->controller->wq, - tmio_nand_dev_ready(mtd), + tmio_nand_dev_ready(nand_chip), msecs_to_jiffies(nand_chip->state == FL_ERASING ? 400 : 20)); - if (unlikely(!tmio_nand_dev_ready(mtd))) { + if (unlikely(!tmio_nand_dev_ready(nand_chip))) { tmio_iowrite8(0x00, tmio->fcr + FCR_IMR); dev_warn(&tmio->dev->dev, "still busy with %s after %d ms\n", nand_chip->state == FL_ERASING ? "erase" : "program", @@ -225,9 +223,9 @@ tmio_nand_wait(struct mtd_info *mtd, struct nand_chip *nand_chip) *To prevent stale data from being read, tmio_nand_hwcontrol() clears *tmio->read_good. */ -static u_char tmio_nand_read_byte(struct mtd_info *mtd) +static u_char tmio_nand_read_byte(struct nand_chip *chip) { - struct tmio_nand *tmio = mtd_to_tmio(mtd); + struct tmio_nand *tmio = mtd_to_tmio(nand_to_mtd(chip)); unsigned int data; if (tmio->read_good--) @@ -245,33 +243,33 @@ static u_char tmio_nand_read_byte(struct mtd_info *mtd) *buffer functions. */ static void -tmio_nand_write_buf(struct mtd_info *mtd, const u_char *buf, int len) +tmio_nand_write_buf(struct nand_chip *chip, const u_char *buf, int len) { - struct tmio_nand *tmio = mtd_to_tmio(mtd); + struct tmio_nand *tmio = mtd_to_tmio(nand_to_mtd(chip)); tmio_iowrite16_rep(tmio->fcr + FCR_DATA, buf, len >> 1); } -static void tmio_nand_read_buf(struct mtd_info *mtd, u_char *buf, int len) +static void tmio_nand_read_buf(struct nand_chip *chip, u_char *buf, int len) { - struct tmio_nand *tmio = mtd_to_tmio(mtd); + struct tmio_nand *tmio = mtd_to_tmio(nand_to_mtd(chip)); tmio_ioread16_rep(tmio->fcr + FCR_DATA, buf, len >> 1); } -static void tmio_nand_enable_hwecc(struct mtd_info *mtd, int mode) +static void tmio_nand_enable_hwecc(struct nand_chip *chip, int mode) { - struct tmio_nand *tmio = mtd_to_tmio(mtd); + struct tmio_nand *tmio = mtd_to_tmio(nand_to_mtd(chip)); tmio_iowrite8(FCR_MODE_HWECC_RESET, tmio->fcr + FCR_MODE); tmio_ioread8(tmio->fcr + FCR_DATA); /* dummy read */ tmio_iowrite8(FCR_MODE_HWECC_CALC, tmio->fcr + FCR_MODE); } -static int tmio_nand_calculate_ecc(struct mtd_info *mtd, const u_char *dat, - u_char *ecc_code) +static int tmio_nand_calculate_ecc(struct nand_chip *chip, const u_char *dat, + u_char *ecc_code) { - struct tmio_nand *tmio = mtd_to_tmio(mtd); + struct tmio_nand *tmio = mtd_to_tmio(nand_to_mtd(chip)); unsigned int ecc; tmio_iowrite8(FCR_MODE_HWECC_RESULT, tmio->fcr + FCR_MODE); @@ -290,16 +288,18 @@ static int tmio_nand_calculate_ecc(struct mtd_info *mtd, const u_char *dat, return 0; } -static int tmio_nand_correct_data(struct mtd_info *mtd, unsigned char *buf, - unsigned char *read_ecc, unsigned char *calc_ecc) +static int tmio_nand_correct_data(struct nand_chip *chip, unsigned char *buf, + unsigned char *read_ecc, + unsigned char *calc_ecc) { int r0, r1; /* assume ecc.size = 512 and ecc.bytes = 6 */ - r0 = __nand_correct_data(buf, read_ecc, calc_ecc, 256); + r0 = __nand_correct_data(buf, read_ecc, calc_ecc, 256, false); if (r0 < 0) return r0; - r1 = __nand_correct_data(buf + 256, read_ecc + 3, calc_ecc + 3, 256); + r1 = __nand_correct_data(buf + 256, read_ecc + 3, calc_ecc + 3, 256, + false); if (r1 < 0) return r1; return r0 + r1; @@ -400,15 +400,15 @@ static int tmio_probe(struct platform_device *dev) return retval; /* Set address of NAND IO lines */ - nand_chip->IO_ADDR_R = tmio->fcr; - nand_chip->IO_ADDR_W = tmio->fcr; + nand_chip->legacy.IO_ADDR_R = tmio->fcr; + nand_chip->legacy.IO_ADDR_W = tmio->fcr; /* Set address of hardware control function */ - nand_chip->cmd_ctrl = tmio_nand_hwcontrol; - nand_chip->dev_ready = tmio_nand_dev_ready; - nand_chip->read_byte = tmio_nand_read_byte; - nand_chip->write_buf = tmio_nand_write_buf; - nand_chip->read_buf = tmio_nand_read_buf; + nand_chip->legacy.cmd_ctrl = tmio_nand_hwcontrol; + nand_chip->legacy.dev_ready = tmio_nand_dev_ready; + nand_chip->legacy.read_byte = tmio_nand_read_byte; + nand_chip->legacy.write_buf = tmio_nand_write_buf; + nand_chip->legacy.read_buf = tmio_nand_read_buf; /* set eccmode using hardware ECC */ nand_chip->ecc.mode = NAND_ECC_HW; @@ -423,7 +423,7 @@ static int tmio_probe(struct platform_device *dev) nand_chip->badblock_pattern = data->badblock_pattern; /* 15 us command delay time */ - nand_chip->chip_delay = 15; + nand_chip->legacy.chip_delay = 15; retval = devm_request_irq(&dev->dev, irq, &tmio_irq, 0, dev_name(&dev->dev), tmio); @@ -433,10 +433,10 @@ static int tmio_probe(struct platform_device *dev) } tmio->irq = irq; - nand_chip->waitfunc = tmio_nand_wait; + nand_chip->legacy.waitfunc = tmio_nand_wait; /* Scan to find existence of the device */ - retval = nand_scan(mtd, 1); + retval = nand_scan(nand_chip, 1); if (retval) goto err_irq; @@ -449,7 +449,7 @@ static int tmio_probe(struct platform_device *dev) if (!retval) return retval; - nand_release(mtd); + nand_release(nand_chip); err_irq: tmio_hw_stop(dev, tmio); @@ -460,7 +460,7 @@ static int tmio_remove(struct platform_device *dev) { struct tmio_nand *tmio = platform_get_drvdata(dev); - nand_release(nand_to_mtd(&tmio->chip)); + nand_release(&tmio->chip); tmio_hw_stop(dev, tmio); return 0; } diff --git a/drivers/mtd/nand/raw/txx9ndfmc.c b/drivers/mtd/nand/raw/txx9ndfmc.c index 4d61a14fcb65..ddf0420c0997 100644 --- a/drivers/mtd/nand/raw/txx9ndfmc.c +++ b/drivers/mtd/nand/raw/txx9ndfmc.c @@ -102,17 +102,17 @@ static void txx9ndfmc_write(struct platform_device *dev, __raw_writel(val, ndregaddr(dev, reg)); } -static uint8_t txx9ndfmc_read_byte(struct mtd_info *mtd) +static uint8_t txx9ndfmc_read_byte(struct nand_chip *chip) { - struct platform_device *dev = mtd_to_platdev(mtd); + struct platform_device *dev = mtd_to_platdev(nand_to_mtd(chip)); return txx9ndfmc_read(dev, TXX9_NDFDTR); } -static void txx9ndfmc_write_buf(struct mtd_info *mtd, const uint8_t *buf, +static void txx9ndfmc_write_buf(struct nand_chip *chip, const uint8_t *buf, int len) { - struct platform_device *dev = mtd_to_platdev(mtd); + struct platform_device *dev = mtd_to_platdev(nand_to_mtd(chip)); void __iomem *ndfdtr = ndregaddr(dev, TXX9_NDFDTR); u32 mcr = txx9ndfmc_read(dev, TXX9_NDFMCR); @@ -122,19 +122,18 @@ static void txx9ndfmc_write_buf(struct mtd_info *mtd, const uint8_t *buf, txx9ndfmc_write(dev, mcr, TXX9_NDFMCR); } -static void txx9ndfmc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) +static void txx9ndfmc_read_buf(struct nand_chip *chip, uint8_t *buf, int len) { - struct platform_device *dev = mtd_to_platdev(mtd); + struct platform_device *dev = mtd_to_platdev(nand_to_mtd(chip)); void __iomem *ndfdtr = ndregaddr(dev, TXX9_NDFDTR); while (len--) *buf++ = __raw_readl(ndfdtr); } -static void txx9ndfmc_cmd_ctrl(struct mtd_info *mtd, int cmd, +static void txx9ndfmc_cmd_ctrl(struct nand_chip *chip, int cmd, unsigned int ctrl) { - struct nand_chip *chip = mtd_to_nand(mtd); struct txx9ndfmc_priv *txx9_priv = nand_get_controller_data(chip); struct platform_device *dev = txx9_priv->dev; struct txx9ndfmc_platform_data *plat = dev_get_platdata(&dev->dev); @@ -163,18 +162,17 @@ static void txx9ndfmc_cmd_ctrl(struct mtd_info *mtd, int cmd, mmiowb(); } -static int txx9ndfmc_dev_ready(struct mtd_info *mtd) +static int txx9ndfmc_dev_ready(struct nand_chip *chip) { - struct platform_device *dev = mtd_to_platdev(mtd); + struct platform_device *dev = mtd_to_platdev(nand_to_mtd(chip)); return !(txx9ndfmc_read(dev, TXX9_NDFSR) & TXX9_NDFSR_BUSY); } -static int txx9ndfmc_calculate_ecc(struct mtd_info *mtd, const uint8_t *dat, +static int txx9ndfmc_calculate_ecc(struct nand_chip *chip, const uint8_t *dat, uint8_t *ecc_code) { - struct platform_device *dev = mtd_to_platdev(mtd); - struct nand_chip *chip = mtd_to_nand(mtd); + struct platform_device *dev = mtd_to_platdev(nand_to_mtd(chip)); int eccbytes; u32 mcr = txx9ndfmc_read(dev, TXX9_NDFMCR); @@ -191,16 +189,17 @@ static int txx9ndfmc_calculate_ecc(struct mtd_info *mtd, const uint8_t *dat, return 0; } -static int txx9ndfmc_correct_data(struct mtd_info *mtd, unsigned char *buf, - unsigned char *read_ecc, unsigned char *calc_ecc) +static int txx9ndfmc_correct_data(struct nand_chip *chip, unsigned char *buf, + unsigned char *read_ecc, + unsigned char *calc_ecc) { - struct nand_chip *chip = mtd_to_nand(mtd); int eccsize; int corrected = 0; int stat; for (eccsize = chip->ecc.size; eccsize > 0; eccsize -= 256) { - stat = __nand_correct_data(buf, read_ecc, calc_ecc, 256); + stat = __nand_correct_data(buf, read_ecc, calc_ecc, 256, + false); if (stat < 0) return stat; corrected += stat; @@ -211,9 +210,9 @@ static int txx9ndfmc_correct_data(struct mtd_info *mtd, unsigned char *buf, return corrected; } -static void txx9ndfmc_enable_hwecc(struct mtd_info *mtd, int mode) +static void txx9ndfmc_enable_hwecc(struct nand_chip *chip, int mode) { - struct platform_device *dev = mtd_to_platdev(mtd); + struct platform_device *dev = mtd_to_platdev(nand_to_mtd(chip)); u32 mcr = txx9ndfmc_read(dev, TXX9_NDFMCR); mcr &= ~TXX9_NDFMCR_ECC_ALL; @@ -326,17 +325,17 @@ static int __init txx9ndfmc_probe(struct platform_device *dev) mtd = nand_to_mtd(chip); mtd->dev.parent = &dev->dev; - chip->read_byte = txx9ndfmc_read_byte; - chip->read_buf = txx9ndfmc_read_buf; - chip->write_buf = txx9ndfmc_write_buf; - chip->cmd_ctrl = txx9ndfmc_cmd_ctrl; - chip->dev_ready = txx9ndfmc_dev_ready; + chip->legacy.read_byte = txx9ndfmc_read_byte; + chip->legacy.read_buf = txx9ndfmc_read_buf; + chip->legacy.write_buf = txx9ndfmc_write_buf; + chip->legacy.cmd_ctrl = txx9ndfmc_cmd_ctrl; + chip->legacy.dev_ready = txx9ndfmc_dev_ready; chip->ecc.calculate = txx9ndfmc_calculate_ecc; chip->ecc.correct = txx9ndfmc_correct_data; chip->ecc.hwctl = txx9ndfmc_enable_hwecc; chip->ecc.mode = NAND_ECC_HW; chip->ecc.strength = 1; - chip->chip_delay = 100; + chip->legacy.chip_delay = 100; chip->controller = &drvdata->controller; nand_set_controller_data(chip, txx9_priv); @@ -359,7 +358,7 @@ static int __init txx9ndfmc_probe(struct platform_device *dev) if (plat->wide_mask & (1 << i)) chip->options |= NAND_BUSWIDTH_16; - if (nand_scan(mtd, 1)) { + if (nand_scan(chip, 1)) { kfree(txx9_priv->mtdname); kfree(txx9_priv); continue; @@ -390,7 +389,7 @@ static int __exit txx9ndfmc_remove(struct platform_device *dev) chip = mtd_to_nand(mtd); txx9_priv = nand_get_controller_data(chip); - nand_release(mtd); + nand_release(chip); kfree(txx9_priv->mtdname); kfree(txx9_priv); } diff --git a/drivers/mtd/nand/raw/vf610_nfc.c b/drivers/mtd/nand/raw/vf610_nfc.c index 6f6dcbf9095b..9814fd4a84cf 100644 --- a/drivers/mtd/nand/raw/vf610_nfc.c +++ b/drivers/mtd/nand/raw/vf610_nfc.c @@ -498,9 +498,9 @@ static int vf610_nfc_exec_op(struct nand_chip *chip, /* * This function supports Vybrid only (MPC5125 would have full RB and four CS) */ -static void vf610_nfc_select_chip(struct mtd_info *mtd, int chip) +static void vf610_nfc_select_chip(struct nand_chip *chip, int cs) { - struct vf610_nfc *nfc = mtd_to_nfc(mtd); + struct vf610_nfc *nfc = mtd_to_nfc(nand_to_mtd(chip)); u32 tmp = vf610_nfc_read(nfc, NFC_ROW_ADDR); /* Vybrid only (MPC5125 would have full RB and four CS) */ @@ -509,9 +509,9 @@ static void vf610_nfc_select_chip(struct mtd_info *mtd, int chip) tmp &= ~(ROW_ADDR_CHIP_SEL_RB_MASK | ROW_ADDR_CHIP_SEL_MASK); - if (chip >= 0) { + if (cs >= 0) { tmp |= 1 << ROW_ADDR_CHIP_SEL_RB_SHIFT; - tmp |= BIT(chip) << ROW_ADDR_CHIP_SEL_SHIFT; + tmp |= BIT(cs) << ROW_ADDR_CHIP_SEL_SHIFT; } vf610_nfc_write(nfc, NFC_ROW_ADDR, tmp); @@ -557,9 +557,10 @@ static void vf610_nfc_fill_row(struct nand_chip *chip, int page, u32 *code, } } -static int vf610_nfc_read_page(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page) +static int vf610_nfc_read_page(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct vf610_nfc *nfc = mtd_to_nfc(mtd); int trfr_sz = mtd->writesize + mtd->oobsize; u32 row = 0, cmd1 = 0, cmd2 = 0, code = 0; @@ -602,9 +603,10 @@ static int vf610_nfc_read_page(struct mtd_info *mtd, struct nand_chip *chip, } } -static int vf610_nfc_write_page(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, int page) +static int vf610_nfc_write_page(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct vf610_nfc *nfc = mtd_to_nfc(mtd); int trfr_sz = mtd->writesize + mtd->oobsize; u32 row = 0, cmd1 = 0, cmd2 = 0, code = 0; @@ -643,24 +645,24 @@ static int vf610_nfc_write_page(struct mtd_info *mtd, struct nand_chip *chip, return 0; } -static int vf610_nfc_read_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, u8 *buf, +static int vf610_nfc_read_page_raw(struct nand_chip *chip, u8 *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct vf610_nfc *nfc = mtd_to_nfc(mtd); int ret; nfc->data_access = true; - ret = nand_read_page_raw(mtd, chip, buf, oob_required, page); + ret = nand_read_page_raw(chip, buf, oob_required, page); nfc->data_access = false; return ret; } -static int vf610_nfc_write_page_raw(struct mtd_info *mtd, - struct nand_chip *chip, const u8 *buf, +static int vf610_nfc_write_page_raw(struct nand_chip *chip, const u8 *buf, int oob_required, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct vf610_nfc *nfc = mtd_to_nfc(mtd); int ret; @@ -677,22 +679,21 @@ static int vf610_nfc_write_page_raw(struct mtd_info *mtd, return nand_prog_page_end_op(chip); } -static int vf610_nfc_read_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int vf610_nfc_read_oob(struct nand_chip *chip, int page) { - struct vf610_nfc *nfc = mtd_to_nfc(mtd); + struct vf610_nfc *nfc = mtd_to_nfc(nand_to_mtd(chip)); int ret; nfc->data_access = true; - ret = nand_read_oob_std(mtd, chip, page); + ret = nand_read_oob_std(chip, page); nfc->data_access = false; return ret; } -static int vf610_nfc_write_oob(struct mtd_info *mtd, struct nand_chip *chip, - int page) +static int vf610_nfc_write_oob(struct nand_chip *chip, int page) { + struct mtd_info *mtd = nand_to_mtd(chip); struct vf610_nfc *nfc = mtd_to_nfc(mtd); int ret; @@ -892,7 +893,7 @@ static int vf610_nfc_probe(struct platform_device *pdev) /* Scan the NAND chip */ chip->dummy_controller.ops = &vf610_nfc_controller_ops; - err = nand_scan(mtd, 1); + err = nand_scan(chip, 1); if (err) goto err_disable_clk; @@ -916,7 +917,7 @@ static int vf610_nfc_remove(struct platform_device *pdev) struct mtd_info *mtd = platform_get_drvdata(pdev); struct vf610_nfc *nfc = mtd_to_nfc(mtd); - nand_release(mtd); + nand_release(mtd_to_nand(mtd)); clk_disable_unprepare(nfc->clk); return 0; } diff --git a/drivers/mtd/nand/raw/xway_nand.c b/drivers/mtd/nand/raw/xway_nand.c index 9926b4e3d69d..a234a5cb4868 100644 --- a/drivers/mtd/nand/raw/xway_nand.c +++ b/drivers/mtd/nand/raw/xway_nand.c @@ -85,9 +85,8 @@ static void xway_writeb(struct mtd_info *mtd, int op, u8 value) writeb(value, data->nandaddr + op); } -static void xway_select_chip(struct mtd_info *mtd, int select) +static void xway_select_chip(struct nand_chip *chip, int select) { - struct nand_chip *chip = mtd_to_nand(mtd); struct xway_nand_data *data = nand_get_controller_data(chip); switch (select) { @@ -106,8 +105,10 @@ static void xway_select_chip(struct mtd_info *mtd, int select) } } -static void xway_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl) +static void xway_cmd_ctrl(struct nand_chip *chip, int cmd, unsigned int ctrl) { + struct mtd_info *mtd = nand_to_mtd(chip); + if (cmd == NAND_CMD_NONE) return; @@ -120,30 +121,30 @@ static void xway_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl) ; } -static int xway_dev_ready(struct mtd_info *mtd) +static int xway_dev_ready(struct nand_chip *chip) { return ltq_ebu_r32(EBU_NAND_WAIT) & NAND_WAIT_RD; } -static unsigned char xway_read_byte(struct mtd_info *mtd) +static unsigned char xway_read_byte(struct nand_chip *chip) { - return xway_readb(mtd, NAND_READ_DATA); + return xway_readb(nand_to_mtd(chip), NAND_READ_DATA); } -static void xway_read_buf(struct mtd_info *mtd, u_char *buf, int len) +static void xway_read_buf(struct nand_chip *chip, u_char *buf, int len) { int i; for (i = 0; i < len; i++) - buf[i] = xway_readb(mtd, NAND_WRITE_DATA); + buf[i] = xway_readb(nand_to_mtd(chip), NAND_WRITE_DATA); } -static void xway_write_buf(struct mtd_info *mtd, const u_char *buf, int len) +static void xway_write_buf(struct nand_chip *chip, const u_char *buf, int len) { int i; for (i = 0; i < len; i++) - xway_writeb(mtd, NAND_WRITE_DATA, buf[i]); + xway_writeb(nand_to_mtd(chip), NAND_WRITE_DATA, buf[i]); } /* @@ -173,13 +174,13 @@ static int xway_nand_probe(struct platform_device *pdev) mtd = nand_to_mtd(&data->chip); mtd->dev.parent = &pdev->dev; - data->chip.cmd_ctrl = xway_cmd_ctrl; - data->chip.dev_ready = xway_dev_ready; + data->chip.legacy.cmd_ctrl = xway_cmd_ctrl; + data->chip.legacy.dev_ready = xway_dev_ready; data->chip.select_chip = xway_select_chip; - data->chip.write_buf = xway_write_buf; - data->chip.read_buf = xway_read_buf; - data->chip.read_byte = xway_read_byte; - data->chip.chip_delay = 30; + data->chip.legacy.write_buf = xway_write_buf; + data->chip.legacy.read_buf = xway_read_buf; + data->chip.legacy.read_byte = xway_read_byte; + data->chip.legacy.chip_delay = 30; data->chip.ecc.mode = NAND_ECC_SOFT; data->chip.ecc.algo = NAND_ECC_HAMMING; @@ -205,13 +206,13 @@ static int xway_nand_probe(struct platform_device *pdev) | cs_flag, EBU_NAND_CON); /* Scan to find existence of the device */ - err = nand_scan(mtd, 1); + err = nand_scan(&data->chip, 1); if (err) return err; err = mtd_device_register(mtd, NULL, 0); if (err) - nand_release(mtd); + nand_release(&data->chip); return err; } @@ -223,7 +224,7 @@ static int xway_nand_remove(struct platform_device *pdev) { struct xway_nand_data *data = platform_get_drvdata(pdev); - nand_release(nand_to_mtd(&data->chip)); + nand_release(&data->chip); return 0; } diff --git a/drivers/mtd/sm_ftl.c b/drivers/mtd/sm_ftl.c index f3bd86e13603..89227b1d036a 100644 --- a/drivers/mtd/sm_ftl.c +++ b/drivers/mtd/sm_ftl.c @@ -221,14 +221,18 @@ static int sm_correct_sector(uint8_t *buffer, struct sm_oob *oob) { uint8_t ecc[3]; - __nand_calculate_ecc(buffer, SM_SMALL_PAGE, ecc); - if (__nand_correct_data(buffer, ecc, oob->ecc1, SM_SMALL_PAGE) < 0) + __nand_calculate_ecc(buffer, SM_SMALL_PAGE, ecc, + IS_ENABLED(CONFIG_MTD_NAND_ECC_SMC)); + if (__nand_correct_data(buffer, ecc, oob->ecc1, SM_SMALL_PAGE, + IS_ENABLED(CONFIG_MTD_NAND_ECC_SMC)) < 0) return -EIO; buffer += SM_SMALL_PAGE; - __nand_calculate_ecc(buffer, SM_SMALL_PAGE, ecc); - if (__nand_correct_data(buffer, ecc, oob->ecc2, SM_SMALL_PAGE) < 0) + __nand_calculate_ecc(buffer, SM_SMALL_PAGE, ecc, + IS_ENABLED(CONFIG_MTD_NAND_ECC_SMC)); + if (__nand_correct_data(buffer, ecc, oob->ecc2, SM_SMALL_PAGE, + IS_ENABLED(CONFIG_MTD_NAND_ECC_SMC)) < 0) return -EIO; return 0; } @@ -393,11 +397,13 @@ restart: } if (ftl->smallpagenand) { - __nand_calculate_ecc(buf + boffset, - SM_SMALL_PAGE, oob.ecc1); + __nand_calculate_ecc(buf + boffset, SM_SMALL_PAGE, + oob.ecc1, + IS_ENABLED(CONFIG_MTD_NAND_ECC_SMC)); __nand_calculate_ecc(buf + boffset + SM_SMALL_PAGE, - SM_SMALL_PAGE, oob.ecc2); + SM_SMALL_PAGE, oob.ecc2, + IS_ENABLED(CONFIG_MTD_NAND_ECC_SMC)); } if (!sm_write_sector(ftl, zone, block, boffset, buf + boffset, &oob)) diff --git a/drivers/mtd/spi-nor/cadence-quadspi.c b/drivers/mtd/spi-nor/cadence-quadspi.c index 8e714fbfa521..e24db817154e 100644 --- a/drivers/mtd/spi-nor/cadence-quadspi.c +++ b/drivers/mtd/spi-nor/cadence-quadspi.c @@ -959,7 +959,7 @@ static int cqspi_direct_read_execute(struct spi_nor *nor, u_char *buf, return 0; } - dma_dst = dma_map_single(nor->dev, buf, len, DMA_DEV_TO_MEM); + dma_dst = dma_map_single(nor->dev, buf, len, DMA_FROM_DEVICE); if (dma_mapping_error(nor->dev, dma_dst)) { dev_err(nor->dev, "dma mapping failed\n"); return -ENOMEM; @@ -994,7 +994,7 @@ static int cqspi_direct_read_execute(struct spi_nor *nor, u_char *buf, } err_unmap: - dma_unmap_single(nor->dev, dma_dst, len, DMA_DEV_TO_MEM); + dma_unmap_single(nor->dev, dma_dst, len, DMA_FROM_DEVICE); return 0; } diff --git a/drivers/mtd/spi-nor/fsl-quadspi.c b/drivers/mtd/spi-nor/fsl-quadspi.c index 7d9620c7ff6c..1ff3430f82c8 100644 --- a/drivers/mtd/spi-nor/fsl-quadspi.c +++ b/drivers/mtd/spi-nor/fsl-quadspi.c @@ -478,6 +478,7 @@ static int fsl_qspi_get_seqid(struct fsl_qspi *q, u8 cmd) { switch (cmd) { case SPINOR_OP_READ_1_1_4: + case SPINOR_OP_READ_1_1_4_4B: return SEQID_READ; case SPINOR_OP_WREN: return SEQID_WREN; @@ -543,6 +544,9 @@ fsl_qspi_runcmd(struct fsl_qspi *q, u8 cmd, unsigned int addr, int len) /* trigger the LUT now */ seqid = fsl_qspi_get_seqid(q, cmd); + if (seqid < 0) + return seqid; + qspi_writel(q, (seqid << QUADSPI_IPCR_SEQID_SHIFT) | len, base + QUADSPI_IPCR); @@ -671,7 +675,7 @@ static void fsl_qspi_set_map_addr(struct fsl_qspi *q) * causes the controller to clear the buffer, and use the sequence pointed * by the QUADSPI_BFGENCR[SEQID] to initiate a read from the flash. */ -static void fsl_qspi_init_ahb_read(struct fsl_qspi *q) +static int fsl_qspi_init_ahb_read(struct fsl_qspi *q) { void __iomem *base = q->iobase; int seqid; @@ -696,8 +700,13 @@ static void fsl_qspi_init_ahb_read(struct fsl_qspi *q) /* Set the default lut sequence for AHB Read. */ seqid = fsl_qspi_get_seqid(q, q->nor[0].read_opcode); + if (seqid < 0) + return seqid; + qspi_writel(q, seqid << QUADSPI_BFGENCR_SEQID_SHIFT, q->iobase + QUADSPI_BFGENCR); + + return 0; } /* This function was used to prepare and enable QSPI clock */ @@ -805,9 +814,7 @@ static int fsl_qspi_nor_setup_last(struct fsl_qspi *q) fsl_qspi_init_lut(q); /* Init for AHB read */ - fsl_qspi_init_ahb_read(q); - - return 0; + return fsl_qspi_init_ahb_read(q); } static const struct of_device_id fsl_qspi_dt_ids[] = { diff --git a/drivers/mtd/spi-nor/intel-spi-pci.c b/drivers/mtd/spi-nor/intel-spi-pci.c index c0976f2e3dd1..872b40922608 100644 --- a/drivers/mtd/spi-nor/intel-spi-pci.c +++ b/drivers/mtd/spi-nor/intel-spi-pci.c @@ -65,6 +65,7 @@ static void intel_spi_pci_remove(struct pci_dev *pdev) static const struct pci_device_id intel_spi_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x18e0), (unsigned long)&bxt_info }, { PCI_VDEVICE(INTEL, 0x19e0), (unsigned long)&bxt_info }, + { PCI_VDEVICE(INTEL, 0x34a4), (unsigned long)&bxt_info }, { PCI_VDEVICE(INTEL, 0xa1a4), (unsigned long)&bxt_info }, { PCI_VDEVICE(INTEL, 0xa224), (unsigned long)&bxt_info }, { }, diff --git a/drivers/mtd/spi-nor/spi-nor.c b/drivers/mtd/spi-nor/spi-nor.c index f028277fb1ce..9407ca5f9443 100644 --- a/drivers/mtd/spi-nor/spi-nor.c +++ b/drivers/mtd/spi-nor/spi-nor.c @@ -18,6 +18,7 @@ #include <linux/math64.h> #include <linux/sizes.h> #include <linux/slab.h> +#include <linux/sort.h> #include <linux/mtd/mtd.h> #include <linux/of_platform.h> @@ -260,6 +261,18 @@ static void spi_nor_set_4byte_opcodes(struct spi_nor *nor, nor->read_opcode = spi_nor_convert_3to4_read(nor->read_opcode); nor->program_opcode = spi_nor_convert_3to4_program(nor->program_opcode); nor->erase_opcode = spi_nor_convert_3to4_erase(nor->erase_opcode); + + if (!spi_nor_has_uniform_erase(nor)) { + struct spi_nor_erase_map *map = &nor->erase_map; + struct spi_nor_erase_type *erase; + int i; + + for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) { + erase = &map->erase_type[i]; + erase->opcode = + spi_nor_convert_3to4_erase(erase->opcode); + } + } } /* Enable/disable 4-byte addressing mode. */ @@ -497,6 +510,277 @@ static int spi_nor_erase_sector(struct spi_nor *nor, u32 addr) return nor->write_reg(nor, nor->erase_opcode, buf, nor->addr_width); } +/** + * spi_nor_div_by_erase_size() - calculate remainder and update new dividend + * @erase: pointer to a structure that describes a SPI NOR erase type + * @dividend: dividend value + * @remainder: pointer to u32 remainder (will be updated) + * + * Return: the result of the division + */ +static u64 spi_nor_div_by_erase_size(const struct spi_nor_erase_type *erase, + u64 dividend, u32 *remainder) +{ + /* JEDEC JESD216B Standard imposes erase sizes to be power of 2. */ + *remainder = (u32)dividend & erase->size_mask; + return dividend >> erase->size_shift; +} + +/** + * spi_nor_find_best_erase_type() - find the best erase type for the given + * offset in the serial flash memory and the + * number of bytes to erase. The region in + * which the address fits is expected to be + * provided. + * @map: the erase map of the SPI NOR + * @region: pointer to a structure that describes a SPI NOR erase region + * @addr: offset in the serial flash memory + * @len: number of bytes to erase + * + * Return: a pointer to the best fitted erase type, NULL otherwise. + */ +static const struct spi_nor_erase_type * +spi_nor_find_best_erase_type(const struct spi_nor_erase_map *map, + const struct spi_nor_erase_region *region, + u64 addr, u32 len) +{ + const struct spi_nor_erase_type *erase; + u32 rem; + int i; + u8 erase_mask = region->offset & SNOR_ERASE_TYPE_MASK; + + /* + * Erase types are ordered by size, with the biggest erase type at + * index 0. + */ + for (i = SNOR_ERASE_TYPE_MAX - 1; i >= 0; i--) { + /* Does the erase region support the tested erase type? */ + if (!(erase_mask & BIT(i))) + continue; + + erase = &map->erase_type[i]; + + /* Don't erase more than what the user has asked for. */ + if (erase->size > len) + continue; + + /* Alignment is not mandatory for overlaid regions */ + if (region->offset & SNOR_OVERLAID_REGION) + return erase; + + spi_nor_div_by_erase_size(erase, addr, &rem); + if (rem) + continue; + else + return erase; + } + + return NULL; +} + +/** + * spi_nor_region_next() - get the next spi nor region + * @region: pointer to a structure that describes a SPI NOR erase region + * + * Return: the next spi nor region or NULL if last region. + */ +static struct spi_nor_erase_region * +spi_nor_region_next(struct spi_nor_erase_region *region) +{ + if (spi_nor_region_is_last(region)) + return NULL; + region++; + return region; +} + +/** + * spi_nor_find_erase_region() - find the region of the serial flash memory in + * which the offset fits + * @map: the erase map of the SPI NOR + * @addr: offset in the serial flash memory + * + * Return: a pointer to the spi_nor_erase_region struct, ERR_PTR(-errno) + * otherwise. + */ +static struct spi_nor_erase_region * +spi_nor_find_erase_region(const struct spi_nor_erase_map *map, u64 addr) +{ + struct spi_nor_erase_region *region = map->regions; + u64 region_start = region->offset & ~SNOR_ERASE_FLAGS_MASK; + u64 region_end = region_start + region->size; + + while (addr < region_start || addr >= region_end) { + region = spi_nor_region_next(region); + if (!region) + return ERR_PTR(-EINVAL); + + region_start = region->offset & ~SNOR_ERASE_FLAGS_MASK; + region_end = region_start + region->size; + } + + return region; +} + +/** + * spi_nor_init_erase_cmd() - initialize an erase command + * @region: pointer to a structure that describes a SPI NOR erase region + * @erase: pointer to a structure that describes a SPI NOR erase type + * + * Return: the pointer to the allocated erase command, ERR_PTR(-errno) + * otherwise. + */ +static struct spi_nor_erase_command * +spi_nor_init_erase_cmd(const struct spi_nor_erase_region *region, + const struct spi_nor_erase_type *erase) +{ + struct spi_nor_erase_command *cmd; + + cmd = kmalloc(sizeof(*cmd), GFP_KERNEL); + if (!cmd) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&cmd->list); + cmd->opcode = erase->opcode; + cmd->count = 1; + + if (region->offset & SNOR_OVERLAID_REGION) + cmd->size = region->size; + else + cmd->size = erase->size; + + return cmd; +} + +/** + * spi_nor_destroy_erase_cmd_list() - destroy erase command list + * @erase_list: list of erase commands + */ +static void spi_nor_destroy_erase_cmd_list(struct list_head *erase_list) +{ + struct spi_nor_erase_command *cmd, *next; + + list_for_each_entry_safe(cmd, next, erase_list, list) { + list_del(&cmd->list); + kfree(cmd); + } +} + +/** + * spi_nor_init_erase_cmd_list() - initialize erase command list + * @nor: pointer to a 'struct spi_nor' + * @erase_list: list of erase commands to be executed once we validate that the + * erase can be performed + * @addr: offset in the serial flash memory + * @len: number of bytes to erase + * + * Builds the list of best fitted erase commands and verifies if the erase can + * be performed. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_init_erase_cmd_list(struct spi_nor *nor, + struct list_head *erase_list, + u64 addr, u32 len) +{ + const struct spi_nor_erase_map *map = &nor->erase_map; + const struct spi_nor_erase_type *erase, *prev_erase = NULL; + struct spi_nor_erase_region *region; + struct spi_nor_erase_command *cmd = NULL; + u64 region_end; + int ret = -EINVAL; + + region = spi_nor_find_erase_region(map, addr); + if (IS_ERR(region)) + return PTR_ERR(region); + + region_end = spi_nor_region_end(region); + + while (len) { + erase = spi_nor_find_best_erase_type(map, region, addr, len); + if (!erase) + goto destroy_erase_cmd_list; + + if (prev_erase != erase || + region->offset & SNOR_OVERLAID_REGION) { + cmd = spi_nor_init_erase_cmd(region, erase); + if (IS_ERR(cmd)) { + ret = PTR_ERR(cmd); + goto destroy_erase_cmd_list; + } + + list_add_tail(&cmd->list, erase_list); + } else { + cmd->count++; + } + + addr += cmd->size; + len -= cmd->size; + + if (len && addr >= region_end) { + region = spi_nor_region_next(region); + if (!region) + goto destroy_erase_cmd_list; + region_end = spi_nor_region_end(region); + } + + prev_erase = erase; + } + + return 0; + +destroy_erase_cmd_list: + spi_nor_destroy_erase_cmd_list(erase_list); + return ret; +} + +/** + * spi_nor_erase_multi_sectors() - perform a non-uniform erase + * @nor: pointer to a 'struct spi_nor' + * @addr: offset in the serial flash memory + * @len: number of bytes to erase + * + * Build a list of best fitted erase commands and execute it once we validate + * that the erase can be performed. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_erase_multi_sectors(struct spi_nor *nor, u64 addr, u32 len) +{ + LIST_HEAD(erase_list); + struct spi_nor_erase_command *cmd, *next; + int ret; + + ret = spi_nor_init_erase_cmd_list(nor, &erase_list, addr, len); + if (ret) + return ret; + + list_for_each_entry_safe(cmd, next, &erase_list, list) { + nor->erase_opcode = cmd->opcode; + while (cmd->count) { + write_enable(nor); + + ret = spi_nor_erase_sector(nor, addr); + if (ret) + goto destroy_erase_cmd_list; + + addr += cmd->size; + cmd->count--; + + ret = spi_nor_wait_till_ready(nor); + if (ret) + goto destroy_erase_cmd_list; + } + list_del(&cmd->list); + kfree(cmd); + } + + return 0; + +destroy_erase_cmd_list: + spi_nor_destroy_erase_cmd_list(&erase_list); + return ret; +} + /* * Erase an address range on the nor chip. The address range may extend * one or more erase sectors. Return an error is there is a problem erasing. @@ -511,9 +795,11 @@ static int spi_nor_erase(struct mtd_info *mtd, struct erase_info *instr) dev_dbg(nor->dev, "at 0x%llx, len %lld\n", (long long)instr->addr, (long long)instr->len); - div_u64_rem(instr->len, mtd->erasesize, &rem); - if (rem) - return -EINVAL; + if (spi_nor_has_uniform_erase(nor)) { + div_u64_rem(instr->len, mtd->erasesize, &rem); + if (rem) + return -EINVAL; + } addr = instr->addr; len = instr->len; @@ -552,7 +838,7 @@ static int spi_nor_erase(struct mtd_info *mtd, struct erase_info *instr) */ /* "sector"-at-a-time erase */ - } else { + } else if (spi_nor_has_uniform_erase(nor)) { while (len) { write_enable(nor); @@ -567,6 +853,12 @@ static int spi_nor_erase(struct mtd_info *mtd, struct erase_info *instr) if (ret) goto erase_err; } + + /* erase multiple sectors */ + } else { + ret = spi_nor_erase_multi_sectors(nor, addr, len); + if (ret) + goto erase_err; } write_disable(nor); @@ -1464,13 +1756,6 @@ static int spi_nor_write(struct mtd_info *mtd, loff_t to, size_t len, goto write_err; *retlen += written; i += written; - if (written != page_remain) { - dev_err(nor->dev, - "While writing %zu bytes written %zd bytes\n", - page_remain, written); - ret = -EIO; - goto write_err; - } } write_err: @@ -1864,6 +2149,36 @@ spi_nor_set_pp_settings(struct spi_nor_pp_command *pp, */ /** + * spi_nor_read_raw() - raw read of serial flash memory. read_opcode, + * addr_width and read_dummy members of the struct spi_nor + * should be previously + * set. + * @nor: pointer to a 'struct spi_nor' + * @addr: offset in the serial flash memory + * @len: number of bytes to read + * @buf: buffer where the data is copied into + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_read_raw(struct spi_nor *nor, u32 addr, size_t len, u8 *buf) +{ + int ret; + + while (len) { + ret = nor->read(nor, addr, len, buf); + if (!ret || ret > len) + return -EIO; + if (ret < 0) + return ret; + + buf += ret; + addr += ret; + len -= ret; + } + return 0; +} + +/** * spi_nor_read_sfdp() - read Serial Flash Discoverable Parameters. * @nor: pointer to a 'struct spi_nor' * @addr: offset in the SFDP area to start reading data from @@ -1890,22 +2205,8 @@ static int spi_nor_read_sfdp(struct spi_nor *nor, u32 addr, nor->addr_width = 3; nor->read_dummy = 8; - while (len) { - ret = nor->read(nor, addr, len, (u8 *)buf); - if (!ret || ret > len) { - ret = -EIO; - goto read_err; - } - if (ret < 0) - goto read_err; - - buf += ret; - addr += ret; - len -= ret; - } - ret = 0; + ret = spi_nor_read_raw(nor, addr, len, buf); -read_err: nor->read_opcode = read_opcode; nor->addr_width = addr_width; nor->read_dummy = read_dummy; @@ -2166,6 +2467,116 @@ static const struct sfdp_bfpt_erase sfdp_bfpt_erases[] = { static int spi_nor_hwcaps_read2cmd(u32 hwcaps); /** + * spi_nor_set_erase_type() - set a SPI NOR erase type + * @erase: pointer to a structure that describes a SPI NOR erase type + * @size: the size of the sector/block erased by the erase type + * @opcode: the SPI command op code to erase the sector/block + */ +static void spi_nor_set_erase_type(struct spi_nor_erase_type *erase, + u32 size, u8 opcode) +{ + erase->size = size; + erase->opcode = opcode; + /* JEDEC JESD216B Standard imposes erase sizes to be power of 2. */ + erase->size_shift = ffs(erase->size) - 1; + erase->size_mask = (1 << erase->size_shift) - 1; +} + +/** + * spi_nor_set_erase_settings_from_bfpt() - set erase type settings from BFPT + * @erase: pointer to a structure that describes a SPI NOR erase type + * @size: the size of the sector/block erased by the erase type + * @opcode: the SPI command op code to erase the sector/block + * @i: erase type index as sorted in the Basic Flash Parameter Table + * + * The supported Erase Types will be sorted at init in ascending order, with + * the smallest Erase Type size being the first member in the erase_type array + * of the spi_nor_erase_map structure. Save the Erase Type index as sorted in + * the Basic Flash Parameter Table since it will be used later on to + * synchronize with the supported Erase Types defined in SFDP optional tables. + */ +static void +spi_nor_set_erase_settings_from_bfpt(struct spi_nor_erase_type *erase, + u32 size, u8 opcode, u8 i) +{ + erase->idx = i; + spi_nor_set_erase_type(erase, size, opcode); +} + +/** + * spi_nor_map_cmp_erase_type() - compare the map's erase types by size + * @l: member in the left half of the map's erase_type array + * @r: member in the right half of the map's erase_type array + * + * Comparison function used in the sort() call to sort in ascending order the + * map's erase types, the smallest erase type size being the first member in the + * sorted erase_type array. + * + * Return: the result of @l->size - @r->size + */ +static int spi_nor_map_cmp_erase_type(const void *l, const void *r) +{ + const struct spi_nor_erase_type *left = l, *right = r; + + return left->size - right->size; +} + +/** + * spi_nor_regions_sort_erase_types() - sort erase types in each region + * @map: the erase map of the SPI NOR + * + * Function assumes that the erase types defined in the erase map are already + * sorted in ascending order, with the smallest erase type size being the first + * member in the erase_type array. It replicates the sort done for the map's + * erase types. Each region's erase bitmask will indicate which erase types are + * supported from the sorted erase types defined in the erase map. + * Sort the all region's erase type at init in order to speed up the process of + * finding the best erase command at runtime. + */ +static void spi_nor_regions_sort_erase_types(struct spi_nor_erase_map *map) +{ + struct spi_nor_erase_region *region = map->regions; + struct spi_nor_erase_type *erase_type = map->erase_type; + int i; + u8 region_erase_mask, sorted_erase_mask; + + while (region) { + region_erase_mask = region->offset & SNOR_ERASE_TYPE_MASK; + + /* Replicate the sort done for the map's erase types. */ + sorted_erase_mask = 0; + for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) + if (erase_type[i].size && + region_erase_mask & BIT(erase_type[i].idx)) + sorted_erase_mask |= BIT(i); + + /* Overwrite erase mask. */ + region->offset = (region->offset & ~SNOR_ERASE_TYPE_MASK) | + sorted_erase_mask; + + region = spi_nor_region_next(region); + } +} + +/** + * spi_nor_init_uniform_erase_map() - Initialize uniform erase map + * @map: the erase map of the SPI NOR + * @erase_mask: bitmask encoding erase types that can erase the entire + * flash memory + * @flash_size: the spi nor flash memory size + */ +static void spi_nor_init_uniform_erase_map(struct spi_nor_erase_map *map, + u8 erase_mask, u64 flash_size) +{ + /* Offset 0 with erase_mask and SNOR_LAST_REGION bit set */ + map->uniform_region.offset = (erase_mask & SNOR_ERASE_TYPE_MASK) | + SNOR_LAST_REGION; + map->uniform_region.size = flash_size; + map->regions = &map->uniform_region; + map->uniform_erase_type = erase_mask; +} + +/** * spi_nor_parse_bfpt() - read and parse the Basic Flash Parameter Table. * @nor: pointer to a 'struct spi_nor' * @bfpt_header: pointer to the 'struct sfdp_parameter_header' describing @@ -2199,12 +2610,14 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor, const struct sfdp_parameter_header *bfpt_header, struct spi_nor_flash_parameter *params) { - struct mtd_info *mtd = &nor->mtd; + struct spi_nor_erase_map *map = &nor->erase_map; + struct spi_nor_erase_type *erase_type = map->erase_type; struct sfdp_bfpt bfpt; size_t len; int i, cmd, err; u32 addr; u16 half; + u8 erase_mask; /* JESD216 Basic Flash Parameter Table length is at least 9 DWORDs. */ if (bfpt_header->length < BFPT_DWORD_MAX_JESD216) @@ -2273,7 +2686,12 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor, spi_nor_set_read_settings_from_bfpt(read, half, rd->proto); } - /* Sector Erase settings. */ + /* + * Sector Erase settings. Reinitialize the uniform erase map using the + * Erase Types defined in the bfpt table. + */ + erase_mask = 0; + memset(&nor->erase_map, 0, sizeof(nor->erase_map)); for (i = 0; i < ARRAY_SIZE(sfdp_bfpt_erases); i++) { const struct sfdp_bfpt_erase *er = &sfdp_bfpt_erases[i]; u32 erasesize; @@ -2288,18 +2706,25 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor, erasesize = 1U << erasesize; opcode = (half >> 8) & 0xff; -#ifdef CONFIG_MTD_SPI_NOR_USE_4K_SECTORS - if (erasesize == SZ_4K) { - nor->erase_opcode = opcode; - mtd->erasesize = erasesize; - break; - } -#endif - if (!mtd->erasesize || mtd->erasesize < erasesize) { - nor->erase_opcode = opcode; - mtd->erasesize = erasesize; - } + erase_mask |= BIT(i); + spi_nor_set_erase_settings_from_bfpt(&erase_type[i], erasesize, + opcode, i); } + spi_nor_init_uniform_erase_map(map, erase_mask, params->size); + /* + * Sort all the map's Erase Types in ascending order with the smallest + * erase size being the first member in the erase_type array. + */ + sort(erase_type, SNOR_ERASE_TYPE_MAX, sizeof(erase_type[0]), + spi_nor_map_cmp_erase_type, NULL); + /* + * Sort the erase types in the uniform region in order to update the + * uniform_erase_type bitmask. The bitmask will be used later on when + * selecting the uniform erase. + */ + spi_nor_regions_sort_erase_types(map); + map->uniform_erase_type = map->uniform_region.offset & + SNOR_ERASE_TYPE_MASK; /* Stop here if not JESD216 rev A or later. */ if (bfpt_header->length < BFPT_DWORD_MAX) @@ -2341,6 +2766,277 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor, return 0; } +#define SMPT_CMD_ADDRESS_LEN_MASK GENMASK(23, 22) +#define SMPT_CMD_ADDRESS_LEN_0 (0x0UL << 22) +#define SMPT_CMD_ADDRESS_LEN_3 (0x1UL << 22) +#define SMPT_CMD_ADDRESS_LEN_4 (0x2UL << 22) +#define SMPT_CMD_ADDRESS_LEN_USE_CURRENT (0x3UL << 22) + +#define SMPT_CMD_READ_DUMMY_MASK GENMASK(19, 16) +#define SMPT_CMD_READ_DUMMY_SHIFT 16 +#define SMPT_CMD_READ_DUMMY(_cmd) \ + (((_cmd) & SMPT_CMD_READ_DUMMY_MASK) >> SMPT_CMD_READ_DUMMY_SHIFT) +#define SMPT_CMD_READ_DUMMY_IS_VARIABLE 0xfUL + +#define SMPT_CMD_READ_DATA_MASK GENMASK(31, 24) +#define SMPT_CMD_READ_DATA_SHIFT 24 +#define SMPT_CMD_READ_DATA(_cmd) \ + (((_cmd) & SMPT_CMD_READ_DATA_MASK) >> SMPT_CMD_READ_DATA_SHIFT) + +#define SMPT_CMD_OPCODE_MASK GENMASK(15, 8) +#define SMPT_CMD_OPCODE_SHIFT 8 +#define SMPT_CMD_OPCODE(_cmd) \ + (((_cmd) & SMPT_CMD_OPCODE_MASK) >> SMPT_CMD_OPCODE_SHIFT) + +#define SMPT_MAP_REGION_COUNT_MASK GENMASK(23, 16) +#define SMPT_MAP_REGION_COUNT_SHIFT 16 +#define SMPT_MAP_REGION_COUNT(_header) \ + ((((_header) & SMPT_MAP_REGION_COUNT_MASK) >> \ + SMPT_MAP_REGION_COUNT_SHIFT) + 1) + +#define SMPT_MAP_ID_MASK GENMASK(15, 8) +#define SMPT_MAP_ID_SHIFT 8 +#define SMPT_MAP_ID(_header) \ + (((_header) & SMPT_MAP_ID_MASK) >> SMPT_MAP_ID_SHIFT) + +#define SMPT_MAP_REGION_SIZE_MASK GENMASK(31, 8) +#define SMPT_MAP_REGION_SIZE_SHIFT 8 +#define SMPT_MAP_REGION_SIZE(_region) \ + (((((_region) & SMPT_MAP_REGION_SIZE_MASK) >> \ + SMPT_MAP_REGION_SIZE_SHIFT) + 1) * 256) + +#define SMPT_MAP_REGION_ERASE_TYPE_MASK GENMASK(3, 0) +#define SMPT_MAP_REGION_ERASE_TYPE(_region) \ + ((_region) & SMPT_MAP_REGION_ERASE_TYPE_MASK) + +#define SMPT_DESC_TYPE_MAP BIT(1) +#define SMPT_DESC_END BIT(0) + +/** + * spi_nor_smpt_addr_width() - return the address width used in the + * configuration detection command. + * @nor: pointer to a 'struct spi_nor' + * @settings: configuration detection command descriptor, dword1 + */ +static u8 spi_nor_smpt_addr_width(const struct spi_nor *nor, const u32 settings) +{ + switch (settings & SMPT_CMD_ADDRESS_LEN_MASK) { + case SMPT_CMD_ADDRESS_LEN_0: + return 0; + case SMPT_CMD_ADDRESS_LEN_3: + return 3; + case SMPT_CMD_ADDRESS_LEN_4: + return 4; + case SMPT_CMD_ADDRESS_LEN_USE_CURRENT: + /* fall through */ + default: + return nor->addr_width; + } +} + +/** + * spi_nor_smpt_read_dummy() - return the configuration detection command read + * latency, in clock cycles. + * @nor: pointer to a 'struct spi_nor' + * @settings: configuration detection command descriptor, dword1 + * + * Return: the number of dummy cycles for an SMPT read + */ +static u8 spi_nor_smpt_read_dummy(const struct spi_nor *nor, const u32 settings) +{ + u8 read_dummy = SMPT_CMD_READ_DUMMY(settings); + + if (read_dummy == SMPT_CMD_READ_DUMMY_IS_VARIABLE) + return nor->read_dummy; + return read_dummy; +} + +/** + * spi_nor_get_map_in_use() - get the configuration map in use + * @nor: pointer to a 'struct spi_nor' + * @smpt: pointer to the sector map parameter table + */ +static const u32 *spi_nor_get_map_in_use(struct spi_nor *nor, const u32 *smpt) +{ + const u32 *ret = NULL; + u32 i, addr; + int err; + u8 addr_width, read_opcode, read_dummy; + u8 read_data_mask, data_byte, map_id; + + addr_width = nor->addr_width; + read_dummy = nor->read_dummy; + read_opcode = nor->read_opcode; + + map_id = 0; + i = 0; + /* Determine if there are any optional Detection Command Descriptors */ + while (!(smpt[i] & SMPT_DESC_TYPE_MAP)) { + read_data_mask = SMPT_CMD_READ_DATA(smpt[i]); + nor->addr_width = spi_nor_smpt_addr_width(nor, smpt[i]); + nor->read_dummy = spi_nor_smpt_read_dummy(nor, smpt[i]); + nor->read_opcode = SMPT_CMD_OPCODE(smpt[i]); + addr = smpt[i + 1]; + + err = spi_nor_read_raw(nor, addr, 1, &data_byte); + if (err) + goto out; + + /* + * Build an index value that is used to select the Sector Map + * Configuration that is currently in use. + */ + map_id = map_id << 1 | !!(data_byte & read_data_mask); + i = i + 2; + } + + /* Find the matching configuration map */ + while (SMPT_MAP_ID(smpt[i]) != map_id) { + if (smpt[i] & SMPT_DESC_END) + goto out; + /* increment the table index to the next map */ + i += SMPT_MAP_REGION_COUNT(smpt[i]) + 1; + } + + ret = smpt + i; + /* fall through */ +out: + nor->addr_width = addr_width; + nor->read_dummy = read_dummy; + nor->read_opcode = read_opcode; + return ret; +} + +/** + * spi_nor_region_check_overlay() - set overlay bit when the region is overlaid + * @region: pointer to a structure that describes a SPI NOR erase region + * @erase: pointer to a structure that describes a SPI NOR erase type + * @erase_type: erase type bitmask + */ +static void +spi_nor_region_check_overlay(struct spi_nor_erase_region *region, + const struct spi_nor_erase_type *erase, + const u8 erase_type) +{ + int i; + + for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) { + if (!(erase_type & BIT(i))) + continue; + if (region->size & erase[i].size_mask) { + spi_nor_region_mark_overlay(region); + return; + } + } +} + +/** + * spi_nor_init_non_uniform_erase_map() - initialize the non-uniform erase map + * @nor: pointer to a 'struct spi_nor' + * @smpt: pointer to the sector map parameter table + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_init_non_uniform_erase_map(struct spi_nor *nor, + const u32 *smpt) +{ + struct spi_nor_erase_map *map = &nor->erase_map; + const struct spi_nor_erase_type *erase = map->erase_type; + struct spi_nor_erase_region *region; + u64 offset; + u32 region_count; + int i, j; + u8 erase_type; + + region_count = SMPT_MAP_REGION_COUNT(*smpt); + /* + * The regions will be freed when the driver detaches from the + * device. + */ + region = devm_kcalloc(nor->dev, region_count, sizeof(*region), + GFP_KERNEL); + if (!region) + return -ENOMEM; + map->regions = region; + + map->uniform_erase_type = 0xff; + offset = 0; + /* Populate regions. */ + for (i = 0; i < region_count; i++) { + j = i + 1; /* index for the region dword */ + region[i].size = SMPT_MAP_REGION_SIZE(smpt[j]); + erase_type = SMPT_MAP_REGION_ERASE_TYPE(smpt[j]); + region[i].offset = offset | erase_type; + + spi_nor_region_check_overlay(®ion[i], erase, erase_type); + + /* + * Save the erase types that are supported in all regions and + * can erase the entire flash memory. + */ + map->uniform_erase_type &= erase_type; + + offset = (region[i].offset & ~SNOR_ERASE_FLAGS_MASK) + + region[i].size; + } + + spi_nor_region_mark_end(®ion[i - 1]); + + return 0; +} + +/** + * spi_nor_parse_smpt() - parse Sector Map Parameter Table + * @nor: pointer to a 'struct spi_nor' + * @smpt_header: sector map parameter table header + * + * This table is optional, but when available, we parse it to identify the + * location and size of sectors within the main data array of the flash memory + * device and to identify which Erase Types are supported by each sector. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_parse_smpt(struct spi_nor *nor, + const struct sfdp_parameter_header *smpt_header) +{ + const u32 *sector_map; + u32 *smpt; + size_t len; + u32 addr; + int i, ret; + + /* Read the Sector Map Parameter Table. */ + len = smpt_header->length * sizeof(*smpt); + smpt = kzalloc(len, GFP_KERNEL); + if (!smpt) + return -ENOMEM; + + addr = SFDP_PARAM_HEADER_PTP(smpt_header); + ret = spi_nor_read_sfdp(nor, addr, len, smpt); + if (ret) + goto out; + + /* Fix endianness of the SMPT DWORDs. */ + for (i = 0; i < smpt_header->length; i++) + smpt[i] = le32_to_cpu(smpt[i]); + + sector_map = spi_nor_get_map_in_use(nor, smpt); + if (!sector_map) { + ret = -EINVAL; + goto out; + } + + ret = spi_nor_init_non_uniform_erase_map(nor, sector_map); + if (ret) + goto out; + + spi_nor_regions_sort_erase_types(&nor->erase_map); + /* fall through */ +out: + kfree(smpt); + return ret; +} + /** * spi_nor_parse_sfdp() - parse the Serial Flash Discoverable Parameters. * @nor: pointer to a 'struct spi_nor' @@ -2435,7 +3131,7 @@ static int spi_nor_parse_sfdp(struct spi_nor *nor, switch (SFDP_PARAM_HEADER_ID(param_header)) { case SFDP_SECTOR_MAP_ID: - dev_info(dev, "non-uniform erase sector maps are not supported yet.\n"); + err = spi_nor_parse_smpt(nor, param_header); break; default: @@ -2455,6 +3151,9 @@ static int spi_nor_init_params(struct spi_nor *nor, const struct flash_info *info, struct spi_nor_flash_parameter *params) { + struct spi_nor_erase_map *map = &nor->erase_map; + u8 i, erase_mask; + /* Set legacy flash parameters as default. */ memset(params, 0, sizeof(*params)); @@ -2494,6 +3193,28 @@ static int spi_nor_init_params(struct spi_nor *nor, spi_nor_set_pp_settings(¶ms->page_programs[SNOR_CMD_PP], SPINOR_OP_PP, SNOR_PROTO_1_1_1); + /* + * Sector Erase settings. Sort Erase Types in ascending order, with the + * smallest erase size starting at BIT(0). + */ + erase_mask = 0; + i = 0; + if (info->flags & SECT_4K_PMC) { + erase_mask |= BIT(i); + spi_nor_set_erase_type(&map->erase_type[i], 4096u, + SPINOR_OP_BE_4K_PMC); + i++; + } else if (info->flags & SECT_4K) { + erase_mask |= BIT(i); + spi_nor_set_erase_type(&map->erase_type[i], 4096u, + SPINOR_OP_BE_4K); + i++; + } + erase_mask |= BIT(i); + spi_nor_set_erase_type(&map->erase_type[i], info->sector_size, + SPINOR_OP_SE); + spi_nor_init_uniform_erase_map(map, erase_mask, params->size); + /* Select the procedure to set the Quad Enable bit. */ if (params->hwcaps.mask & (SNOR_HWCAPS_READ_QUAD | SNOR_HWCAPS_PP_QUAD)) { @@ -2521,20 +3242,20 @@ static int spi_nor_init_params(struct spi_nor *nor, params->quad_enable = info->quad_enable; } - /* Override the parameters with data read from SFDP tables. */ - nor->addr_width = 0; - nor->mtd.erasesize = 0; if ((info->flags & (SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ)) && !(info->flags & SPI_NOR_SKIP_SFDP)) { struct spi_nor_flash_parameter sfdp_params; + struct spi_nor_erase_map prev_map; memcpy(&sfdp_params, params, sizeof(sfdp_params)); - if (spi_nor_parse_sfdp(nor, &sfdp_params)) { - nor->addr_width = 0; - nor->mtd.erasesize = 0; - } else { + memcpy(&prev_map, &nor->erase_map, sizeof(prev_map)); + + if (spi_nor_parse_sfdp(nor, &sfdp_params)) + /* restore previous erase map */ + memcpy(&nor->erase_map, &prev_map, + sizeof(nor->erase_map)); + else memcpy(params, &sfdp_params, sizeof(*params)); - } } return 0; @@ -2643,29 +3364,103 @@ static int spi_nor_select_pp(struct spi_nor *nor, return 0; } -static int spi_nor_select_erase(struct spi_nor *nor, - const struct flash_info *info) +/** + * spi_nor_select_uniform_erase() - select optimum uniform erase type + * @map: the erase map of the SPI NOR + * @wanted_size: the erase type size to search for. Contains the value of + * info->sector_size or of the "small sector" size in case + * CONFIG_MTD_SPI_NOR_USE_4K_SECTORS is defined. + * + * Once the optimum uniform sector erase command is found, disable all the + * other. + * + * Return: pointer to erase type on success, NULL otherwise. + */ +static const struct spi_nor_erase_type * +spi_nor_select_uniform_erase(struct spi_nor_erase_map *map, + const u32 wanted_size) { - struct mtd_info *mtd = &nor->mtd; + const struct spi_nor_erase_type *tested_erase, *erase = NULL; + int i; + u8 uniform_erase_type = map->uniform_erase_type; - /* Do nothing if already configured from SFDP. */ - if (mtd->erasesize) - return 0; + for (i = SNOR_ERASE_TYPE_MAX - 1; i >= 0; i--) { + if (!(uniform_erase_type & BIT(i))) + continue; + + tested_erase = &map->erase_type[i]; + + /* + * If the current erase size is the one, stop here: + * we have found the right uniform Sector Erase command. + */ + if (tested_erase->size == wanted_size) { + erase = tested_erase; + break; + } + + /* + * Otherwise, the current erase size is still a valid canditate. + * Select the biggest valid candidate. + */ + if (!erase && tested_erase->size) + erase = tested_erase; + /* keep iterating to find the wanted_size */ + } + + if (!erase) + return NULL; + /* Disable all other Sector Erase commands. */ + map->uniform_erase_type &= ~SNOR_ERASE_TYPE_MASK; + map->uniform_erase_type |= BIT(erase - map->erase_type); + return erase; +} + +static int spi_nor_select_erase(struct spi_nor *nor, u32 wanted_size) +{ + struct spi_nor_erase_map *map = &nor->erase_map; + const struct spi_nor_erase_type *erase = NULL; + struct mtd_info *mtd = &nor->mtd; + int i; + + /* + * The previous implementation handling Sector Erase commands assumed + * that the SPI flash memory has an uniform layout then used only one + * of the supported erase sizes for all Sector Erase commands. + * So to be backward compatible, the new implementation also tries to + * manage the SPI flash memory as uniform with a single erase sector + * size, when possible. + */ #ifdef CONFIG_MTD_SPI_NOR_USE_4K_SECTORS /* prefer "small sector" erase if possible */ - if (info->flags & SECT_4K) { - nor->erase_opcode = SPINOR_OP_BE_4K; - mtd->erasesize = 4096; - } else if (info->flags & SECT_4K_PMC) { - nor->erase_opcode = SPINOR_OP_BE_4K_PMC; - mtd->erasesize = 4096; - } else + wanted_size = 4096u; #endif - { - nor->erase_opcode = SPINOR_OP_SE; - mtd->erasesize = info->sector_size; + + if (spi_nor_has_uniform_erase(nor)) { + erase = spi_nor_select_uniform_erase(map, wanted_size); + if (!erase) + return -EINVAL; + nor->erase_opcode = erase->opcode; + mtd->erasesize = erase->size; + return 0; } + + /* + * For non-uniform SPI flash memory, set mtd->erasesize to the + * maximum erase sector size. No need to set nor->erase_opcode. + */ + for (i = SNOR_ERASE_TYPE_MAX - 1; i >= 0; i--) { + if (map->erase_type[i].size) { + erase = &map->erase_type[i]; + break; + } + } + + if (!erase) + return -EINVAL; + + mtd->erasesize = erase->size; return 0; } @@ -2712,7 +3507,7 @@ static int spi_nor_setup(struct spi_nor *nor, const struct flash_info *info, } /* Select the Sector Erase command. */ - err = spi_nor_select_erase(nor, info); + err = spi_nor_select_erase(nor, info->sector_size); if (err) { dev_err(nor->dev, "can't select erase settings supported by both the SPI controller and memory.\n"); diff --git a/drivers/mtd/tests/mtd_nandecctest.c b/drivers/mtd/tests/mtd_nandecctest.c index 88b6c81cebbe..c71523e94580 100644 --- a/drivers/mtd/tests/mtd_nandecctest.c +++ b/drivers/mtd/tests/mtd_nandecctest.c @@ -121,8 +121,10 @@ static int no_bit_error_verify(void *error_data, void *error_ecc, unsigned char calc_ecc[3]; int ret; - __nand_calculate_ecc(error_data, size, calc_ecc); - ret = __nand_correct_data(error_data, error_ecc, calc_ecc, size); + __nand_calculate_ecc(error_data, size, calc_ecc, + IS_ENABLED(CONFIG_MTD_NAND_ECC_SMC)); + ret = __nand_correct_data(error_data, error_ecc, calc_ecc, size, + IS_ENABLED(CONFIG_MTD_NAND_ECC_SMC)); if (ret == 0 && !memcmp(correct_data, error_data, size)) return 0; @@ -149,8 +151,10 @@ static int single_bit_error_correct(void *error_data, void *error_ecc, unsigned char calc_ecc[3]; int ret; - __nand_calculate_ecc(error_data, size, calc_ecc); - ret = __nand_correct_data(error_data, error_ecc, calc_ecc, size); + __nand_calculate_ecc(error_data, size, calc_ecc, + IS_ENABLED(CONFIG_MTD_NAND_ECC_SMC)); + ret = __nand_correct_data(error_data, error_ecc, calc_ecc, size, + IS_ENABLED(CONFIG_MTD_NAND_ECC_SMC)); if (ret == 1 && !memcmp(correct_data, error_data, size)) return 0; @@ -184,8 +188,10 @@ static int double_bit_error_detect(void *error_data, void *error_ecc, unsigned char calc_ecc[3]; int ret; - __nand_calculate_ecc(error_data, size, calc_ecc); - ret = __nand_correct_data(error_data, error_ecc, calc_ecc, size); + __nand_calculate_ecc(error_data, size, calc_ecc, + IS_ENABLED(CONFIG_MTD_NAND_ECC_SMC)); + ret = __nand_correct_data(error_data, error_ecc, calc_ecc, size, + IS_ENABLED(CONFIG_MTD_NAND_ECC_SMC)); return (ret == -EBADMSG) ? 0 : -EINVAL; } @@ -259,7 +265,8 @@ static int nand_ecc_test_run(const size_t size) } prandom_bytes(correct_data, size); - __nand_calculate_ecc(correct_data, size, correct_ecc); + __nand_calculate_ecc(correct_data, size, correct_ecc, + IS_ENABLED(CONFIG_MTD_NAND_ECC_SMC)); for (i = 0; i < ARRAY_SIZE(nand_ecc_test); i++) { nand_ecc_test[i].prepare(error_data, error_ecc, diff --git a/drivers/net/ethernet/broadcom/genet/bcmmii.c b/drivers/net/ethernet/broadcom/genet/bcmmii.c index 4241ae928d4a..34af5f1569c8 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmmii.c +++ b/drivers/net/ethernet/broadcom/genet/bcmmii.c @@ -321,9 +321,12 @@ int bcmgenet_mii_probe(struct net_device *dev) phydev->advertising = phydev->supported; /* The internal PHY has its link interrupts routed to the - * Ethernet MAC ISRs + * Ethernet MAC ISRs. On GENETv5 there is a hardware issue + * that prevents the signaling of link UP interrupts when + * the link operates at 10Mbps, so fallback to polling for + * those versions of GENET. */ - if (priv->internal_phy) + if (priv->internal_phy && !GENET_IS_V5(priv)) dev->phydev->irq = PHY_IGNORE_INTERRUPT; return 0; diff --git a/drivers/net/ethernet/freescale/fec.h b/drivers/net/ethernet/freescale/fec.h index 4778b663653e..bf80855dd0dd 100644 --- a/drivers/net/ethernet/freescale/fec.h +++ b/drivers/net/ethernet/freescale/fec.h @@ -452,6 +452,10 @@ struct bufdesc_ex { * initialisation. */ #define FEC_QUIRK_MIB_CLEAR (1 << 15) +/* Only i.MX25/i.MX27/i.MX28 controller supports FRBR,FRSR registers, + * those FIFO receive registers are resolved in other platforms. + */ +#define FEC_QUIRK_HAS_FRREG (1 << 16) struct bufdesc_prop { int qid; diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index bf9b9fd6d2a0..7b98bb75ba8a 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -91,14 +91,16 @@ static struct platform_device_id fec_devtype[] = { .driver_data = 0, }, { .name = "imx25-fec", - .driver_data = FEC_QUIRK_USE_GASKET | FEC_QUIRK_MIB_CLEAR, + .driver_data = FEC_QUIRK_USE_GASKET | FEC_QUIRK_MIB_CLEAR | + FEC_QUIRK_HAS_FRREG, }, { .name = "imx27-fec", - .driver_data = FEC_QUIRK_MIB_CLEAR, + .driver_data = FEC_QUIRK_MIB_CLEAR | FEC_QUIRK_HAS_FRREG, }, { .name = "imx28-fec", .driver_data = FEC_QUIRK_ENET_MAC | FEC_QUIRK_SWAP_FRAME | - FEC_QUIRK_SINGLE_MDIO | FEC_QUIRK_HAS_RACC, + FEC_QUIRK_SINGLE_MDIO | FEC_QUIRK_HAS_RACC | + FEC_QUIRK_HAS_FRREG, }, { .name = "imx6q-fec", .driver_data = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_GBIT | @@ -2164,7 +2166,13 @@ static void fec_enet_get_regs(struct net_device *ndev, memset(buf, 0, regs->len); for (i = 0; i < ARRAY_SIZE(fec_enet_register_offset); i++) { - off = fec_enet_register_offset[i] / 4; + off = fec_enet_register_offset[i]; + + if ((off == FEC_R_BOUND || off == FEC_R_FSTART) && + !(fep->quirks & FEC_QUIRK_HAS_FRREG)) + continue; + + off >>= 2; buf[off] = readl(&theregs[off]); } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 15d8ae28c040..00172dee5339 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -432,10 +432,9 @@ static inline u16 mlx5e_icosq_wrap_cnt(struct mlx5e_icosq *sq) static inline void mlx5e_fill_icosq_frag_edge(struct mlx5e_icosq *sq, struct mlx5_wq_cyc *wq, - u16 pi, u16 frag_pi) + u16 pi, u16 nnops) { struct mlx5e_sq_wqe_info *edge_wi, *wi = &sq->db.ico_wqe[pi]; - u8 nnops = mlx5_wq_cyc_get_frag_size(wq) - frag_pi; edge_wi = wi + nnops; @@ -454,15 +453,14 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) struct mlx5_wq_cyc *wq = &sq->wq; struct mlx5e_umr_wqe *umr_wqe; u16 xlt_offset = ix << (MLX5E_LOG_ALIGNED_MPWQE_PPW - 1); - u16 pi, frag_pi; + u16 pi, contig_wqebbs_room; int err; int i; pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); - frag_pi = mlx5_wq_cyc_ctr2fragix(wq, sq->pc); - - if (unlikely(frag_pi + MLX5E_UMR_WQEBBS > mlx5_wq_cyc_get_frag_size(wq))) { - mlx5e_fill_icosq_frag_edge(sq, wq, pi, frag_pi); + contig_wqebbs_room = mlx5_wq_cyc_get_contig_wqebbs(wq, pi); + if (unlikely(contig_wqebbs_room < MLX5E_UMR_WQEBBS)) { + mlx5e_fill_icosq_frag_edge(sq, wq, pi, contig_wqebbs_room); pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index ae73ea992845..6dacaeba2fbf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -290,10 +290,9 @@ dma_unmap_wqe_err: static inline void mlx5e_fill_sq_frag_edge(struct mlx5e_txqsq *sq, struct mlx5_wq_cyc *wq, - u16 pi, u16 frag_pi) + u16 pi, u16 nnops) { struct mlx5e_tx_wqe_info *edge_wi, *wi = &sq->db.wqe_info[pi]; - u8 nnops = mlx5_wq_cyc_get_frag_size(wq) - frag_pi; edge_wi = wi + nnops; @@ -348,8 +347,8 @@ netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, struct mlx5e_tx_wqe_info *wi; struct mlx5e_sq_stats *stats = sq->stats; + u16 headlen, ihs, contig_wqebbs_room; u16 ds_cnt, ds_cnt_inl = 0; - u16 headlen, ihs, frag_pi; u8 num_wqebbs, opcode; u32 num_bytes; int num_dma; @@ -386,9 +385,9 @@ netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, } num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); - frag_pi = mlx5_wq_cyc_ctr2fragix(wq, sq->pc); - if (unlikely(frag_pi + num_wqebbs > mlx5_wq_cyc_get_frag_size(wq))) { - mlx5e_fill_sq_frag_edge(sq, wq, pi, frag_pi); + contig_wqebbs_room = mlx5_wq_cyc_get_contig_wqebbs(wq, pi); + if (unlikely(contig_wqebbs_room < num_wqebbs)) { + mlx5e_fill_sq_frag_edge(sq, wq, pi, contig_wqebbs_room); mlx5e_sq_fetch_wqe(sq, &wqe, &pi); } @@ -636,7 +635,7 @@ netdev_tx_t mlx5i_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, struct mlx5e_tx_wqe_info *wi; struct mlx5e_sq_stats *stats = sq->stats; - u16 headlen, ihs, pi, frag_pi; + u16 headlen, ihs, pi, contig_wqebbs_room; u16 ds_cnt, ds_cnt_inl = 0; u8 num_wqebbs, opcode; u32 num_bytes; @@ -672,13 +671,14 @@ netdev_tx_t mlx5i_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, } num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); - frag_pi = mlx5_wq_cyc_ctr2fragix(wq, sq->pc); - if (unlikely(frag_pi + num_wqebbs > mlx5_wq_cyc_get_frag_size(wq))) { + pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); + contig_wqebbs_room = mlx5_wq_cyc_get_contig_wqebbs(wq, pi); + if (unlikely(contig_wqebbs_room < num_wqebbs)) { + mlx5e_fill_sq_frag_edge(sq, wq, pi, contig_wqebbs_room); pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); - mlx5e_fill_sq_frag_edge(sq, wq, pi, frag_pi); } - mlx5i_sq_fetch_wqe(sq, &wqe, &pi); + mlx5i_sq_fetch_wqe(sq, &wqe, pi); /* fill wqe */ wi = &sq->db.wqe_info[pi]; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index 48864f4988a4..c1e1a16a9b07 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -273,7 +273,7 @@ static void eq_pf_process(struct mlx5_eq *eq) case MLX5_PFAULT_SUBTYPE_WQE: /* WQE based event */ pfault->type = - be32_to_cpu(pf_eqe->wqe.pftype_wq) >> 24; + (be32_to_cpu(pf_eqe->wqe.pftype_wq) >> 24) & 0x7; pfault->token = be32_to_cpu(pf_eqe->wqe.token); pfault->wqe.wq_num = diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c index 5645a4facad2..b8ee9101c506 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c @@ -245,7 +245,7 @@ static void *mlx5_fpga_ipsec_cmd_exec(struct mlx5_core_dev *mdev, return ERR_PTR(res); } - /* Context will be freed by wait func after completion */ + /* Context should be freed by the caller after completion. */ return context; } @@ -418,10 +418,8 @@ static int mlx5_fpga_ipsec_set_caps(struct mlx5_core_dev *mdev, u32 flags) cmd.cmd = htonl(MLX5_FPGA_IPSEC_CMD_OP_SET_CAP); cmd.flags = htonl(flags); context = mlx5_fpga_ipsec_cmd_exec(mdev, &cmd, sizeof(cmd)); - if (IS_ERR(context)) { - err = PTR_ERR(context); - goto out; - } + if (IS_ERR(context)) + return PTR_ERR(context); err = mlx5_fpga_ipsec_cmd_wait(context); if (err) @@ -435,6 +433,7 @@ static int mlx5_fpga_ipsec_set_caps(struct mlx5_core_dev *mdev, u32 flags) } out: + kfree(context); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h index 08eac92fc26c..0982c579ec74 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h @@ -109,12 +109,11 @@ struct mlx5i_tx_wqe { static inline void mlx5i_sq_fetch_wqe(struct mlx5e_txqsq *sq, struct mlx5i_tx_wqe **wqe, - u16 *pi) + u16 pi) { struct mlx5_wq_cyc *wq = &sq->wq; - *pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); - *wqe = mlx5_wq_cyc_get_wqe(wq, *pi); + *wqe = mlx5_wq_cyc_get_wqe(wq, pi); memset(*wqe, 0, sizeof(**wqe)); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wq.c b/drivers/net/ethernet/mellanox/mlx5/core/wq.c index 68e7f8df2a6d..ddca327e8950 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/wq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/wq.c @@ -39,11 +39,6 @@ u32 mlx5_wq_cyc_get_size(struct mlx5_wq_cyc *wq) return (u32)wq->fbc.sz_m1 + 1; } -u16 mlx5_wq_cyc_get_frag_size(struct mlx5_wq_cyc *wq) -{ - return wq->fbc.frag_sz_m1 + 1; -} - u32 mlx5_cqwq_get_size(struct mlx5_cqwq *wq) { return wq->fbc.sz_m1 + 1; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wq.h b/drivers/net/ethernet/mellanox/mlx5/core/wq.h index 3a1a170bb2d7..b1293d153a58 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/wq.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/wq.h @@ -80,7 +80,6 @@ int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, void *wqc, struct mlx5_wq_cyc *wq, struct mlx5_wq_ctrl *wq_ctrl); u32 mlx5_wq_cyc_get_size(struct mlx5_wq_cyc *wq); -u16 mlx5_wq_cyc_get_frag_size(struct mlx5_wq_cyc *wq); int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, void *qpc, struct mlx5_wq_qp *wq, @@ -140,11 +139,6 @@ static inline u16 mlx5_wq_cyc_ctr2ix(struct mlx5_wq_cyc *wq, u16 ctr) return ctr & wq->fbc.sz_m1; } -static inline u16 mlx5_wq_cyc_ctr2fragix(struct mlx5_wq_cyc *wq, u16 ctr) -{ - return ctr & wq->fbc.frag_sz_m1; -} - static inline u16 mlx5_wq_cyc_get_head(struct mlx5_wq_cyc *wq) { return mlx5_wq_cyc_ctr2ix(wq, wq->wqe_ctr); @@ -160,6 +154,11 @@ static inline void *mlx5_wq_cyc_get_wqe(struct mlx5_wq_cyc *wq, u16 ix) return mlx5_frag_buf_get_wqe(&wq->fbc, ix); } +static inline u16 mlx5_wq_cyc_get_contig_wqebbs(struct mlx5_wq_cyc *wq, u16 ix) +{ + return mlx5_frag_buf_get_idx_last_contig_stride(&wq->fbc, ix) - ix + 1; +} + static inline int mlx5_wq_cyc_cc_bigger(u16 cc1, u16 cc2) { int equal = (cc1 == cc2); diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index 81533d7f395c..937d0ace699a 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -1055,6 +1055,7 @@ int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, err_driver_init: mlxsw_thermal_fini(mlxsw_core->thermal); err_thermal_init: + mlxsw_hwmon_fini(mlxsw_core->hwmon); err_hwmon_init: if (!reload) devlink_unregister(devlink); @@ -1088,6 +1089,7 @@ void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core, if (mlxsw_core->driver->fini) mlxsw_core->driver->fini(mlxsw_core); mlxsw_thermal_fini(mlxsw_core->thermal); + mlxsw_hwmon_fini(mlxsw_core->hwmon); if (!reload) devlink_unregister(devlink); mlxsw_emad_fini(mlxsw_core); diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h index 655ddd204ab2..c35be477856f 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core.h @@ -359,6 +359,10 @@ static inline int mlxsw_hwmon_init(struct mlxsw_core *mlxsw_core, return 0; } +static inline void mlxsw_hwmon_fini(struct mlxsw_hwmon *mlxsw_hwmon) +{ +} + #endif struct mlxsw_thermal; diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_hwmon.c b/drivers/net/ethernet/mellanox/mlxsw/core_hwmon.c index f6cf2896d337..e04e8162aa14 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_hwmon.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_hwmon.c @@ -303,8 +303,7 @@ int mlxsw_hwmon_init(struct mlxsw_core *mlxsw_core, struct device *hwmon_dev; int err; - mlxsw_hwmon = devm_kzalloc(mlxsw_bus_info->dev, sizeof(*mlxsw_hwmon), - GFP_KERNEL); + mlxsw_hwmon = kzalloc(sizeof(*mlxsw_hwmon), GFP_KERNEL); if (!mlxsw_hwmon) return -ENOMEM; mlxsw_hwmon->core = mlxsw_core; @@ -321,10 +320,9 @@ int mlxsw_hwmon_init(struct mlxsw_core *mlxsw_core, mlxsw_hwmon->groups[0] = &mlxsw_hwmon->group; mlxsw_hwmon->group.attrs = mlxsw_hwmon->attrs; - hwmon_dev = devm_hwmon_device_register_with_groups(mlxsw_bus_info->dev, - "mlxsw", - mlxsw_hwmon, - mlxsw_hwmon->groups); + hwmon_dev = hwmon_device_register_with_groups(mlxsw_bus_info->dev, + "mlxsw", mlxsw_hwmon, + mlxsw_hwmon->groups); if (IS_ERR(hwmon_dev)) { err = PTR_ERR(hwmon_dev); goto err_hwmon_register; @@ -337,5 +335,12 @@ int mlxsw_hwmon_init(struct mlxsw_core *mlxsw_core, err_hwmon_register: err_fans_init: err_temp_init: + kfree(mlxsw_hwmon); return err; } + +void mlxsw_hwmon_fini(struct mlxsw_hwmon *mlxsw_hwmon) +{ + hwmon_device_unregister(mlxsw_hwmon->hwmon_dev); + kfree(mlxsw_hwmon); +} diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c index 1a4f2bb48ead..ed4e298cd823 100644 --- a/drivers/net/ethernet/mscc/ocelot.c +++ b/drivers/net/ethernet/mscc/ocelot.c @@ -133,9 +133,9 @@ static inline int ocelot_vlant_wait_for_completion(struct ocelot *ocelot) { unsigned int val, timeout = 10; - /* Wait for the issued mac table command to be completed, or timeout. - * When the command read from ANA_TABLES_MACACCESS is - * MACACCESS_CMD_IDLE, the issued command completed successfully. + /* Wait for the issued vlan table command to be completed, or timeout. + * When the command read from ANA_TABLES_VLANACCESS is + * VLANACCESS_CMD_IDLE, the issued command completed successfully. */ do { val = ocelot_read(ocelot, ANA_TABLES_VLANACCESS); diff --git a/drivers/net/ethernet/netronome/nfp/flower/action.c b/drivers/net/ethernet/netronome/nfp/flower/action.c index 46ba0cf257c6..7a1e9cd9cc62 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/action.c +++ b/drivers/net/ethernet/netronome/nfp/flower/action.c @@ -429,12 +429,14 @@ nfp_fl_set_ip4(const struct tc_action *action, int idx, u32 off, switch (off) { case offsetof(struct iphdr, daddr): - set_ip_addr->ipv4_dst_mask = mask; - set_ip_addr->ipv4_dst = exact; + set_ip_addr->ipv4_dst_mask |= mask; + set_ip_addr->ipv4_dst &= ~mask; + set_ip_addr->ipv4_dst |= exact & mask; break; case offsetof(struct iphdr, saddr): - set_ip_addr->ipv4_src_mask = mask; - set_ip_addr->ipv4_src = exact; + set_ip_addr->ipv4_src_mask |= mask; + set_ip_addr->ipv4_src &= ~mask; + set_ip_addr->ipv4_src |= exact & mask; break; default: return -EOPNOTSUPP; @@ -448,11 +450,12 @@ nfp_fl_set_ip4(const struct tc_action *action, int idx, u32 off, } static void -nfp_fl_set_ip6_helper(int opcode_tag, int idx, __be32 exact, __be32 mask, +nfp_fl_set_ip6_helper(int opcode_tag, u8 word, __be32 exact, __be32 mask, struct nfp_fl_set_ipv6_addr *ip6) { - ip6->ipv6[idx % 4].mask = mask; - ip6->ipv6[idx % 4].exact = exact; + ip6->ipv6[word].mask |= mask; + ip6->ipv6[word].exact &= ~mask; + ip6->ipv6[word].exact |= exact & mask; ip6->reserved = cpu_to_be16(0); ip6->head.jump_id = opcode_tag; @@ -465,6 +468,7 @@ nfp_fl_set_ip6(const struct tc_action *action, int idx, u32 off, struct nfp_fl_set_ipv6_addr *ip_src) { __be32 exact, mask; + u8 word; /* We are expecting tcf_pedit to return a big endian value */ mask = (__force __be32)~tcf_pedit_mask(action, idx); @@ -473,17 +477,20 @@ nfp_fl_set_ip6(const struct tc_action *action, int idx, u32 off, if (exact & ~mask) return -EOPNOTSUPP; - if (off < offsetof(struct ipv6hdr, saddr)) + if (off < offsetof(struct ipv6hdr, saddr)) { return -EOPNOTSUPP; - else if (off < offsetof(struct ipv6hdr, daddr)) - nfp_fl_set_ip6_helper(NFP_FL_ACTION_OPCODE_SET_IPV6_SRC, idx, + } else if (off < offsetof(struct ipv6hdr, daddr)) { + word = (off - offsetof(struct ipv6hdr, saddr)) / sizeof(exact); + nfp_fl_set_ip6_helper(NFP_FL_ACTION_OPCODE_SET_IPV6_SRC, word, exact, mask, ip_src); - else if (off < offsetof(struct ipv6hdr, daddr) + - sizeof(struct in6_addr)) - nfp_fl_set_ip6_helper(NFP_FL_ACTION_OPCODE_SET_IPV6_DST, idx, + } else if (off < offsetof(struct ipv6hdr, daddr) + + sizeof(struct in6_addr)) { + word = (off - offsetof(struct ipv6hdr, daddr)) / sizeof(exact); + nfp_fl_set_ip6_helper(NFP_FL_ACTION_OPCODE_SET_IPV6_DST, word, exact, mask, ip_dst); - else + } else { return -EOPNOTSUPP; + } return 0; } @@ -541,7 +548,7 @@ nfp_fl_pedit(const struct tc_action *action, struct tc_cls_flower_offload *flow, struct nfp_fl_set_eth set_eth; enum pedit_header_type htype; int idx, nkeys, err; - size_t act_size; + size_t act_size = 0; u32 offset, cmd; u8 ip_proto = 0; @@ -599,7 +606,9 @@ nfp_fl_pedit(const struct tc_action *action, struct tc_cls_flower_offload *flow, act_size = sizeof(set_eth); memcpy(nfp_action, &set_eth, act_size); *a_len += act_size; - } else if (set_ip_addr.head.len_lw) { + } + if (set_ip_addr.head.len_lw) { + nfp_action += act_size; act_size = sizeof(set_ip_addr); memcpy(nfp_action, &set_ip_addr, act_size); *a_len += act_size; @@ -607,10 +616,12 @@ nfp_fl_pedit(const struct tc_action *action, struct tc_cls_flower_offload *flow, /* Hardware will automatically fix IPv4 and TCP/UDP checksum. */ *csum_updated |= TCA_CSUM_UPDATE_FLAG_IPV4HDR | nfp_fl_csum_l4_to_flag(ip_proto); - } else if (set_ip6_dst.head.len_lw && set_ip6_src.head.len_lw) { + } + if (set_ip6_dst.head.len_lw && set_ip6_src.head.len_lw) { /* TC compiles set src and dst IPv6 address as a single action, * the hardware requires this to be 2 separate actions. */ + nfp_action += act_size; act_size = sizeof(set_ip6_src); memcpy(nfp_action, &set_ip6_src, act_size); *a_len += act_size; @@ -623,6 +634,7 @@ nfp_fl_pedit(const struct tc_action *action, struct tc_cls_flower_offload *flow, /* Hardware will automatically fix TCP/UDP checksum. */ *csum_updated |= nfp_fl_csum_l4_to_flag(ip_proto); } else if (set_ip6_dst.head.len_lw) { + nfp_action += act_size; act_size = sizeof(set_ip6_dst); memcpy(nfp_action, &set_ip6_dst, act_size); *a_len += act_size; @@ -630,13 +642,16 @@ nfp_fl_pedit(const struct tc_action *action, struct tc_cls_flower_offload *flow, /* Hardware will automatically fix TCP/UDP checksum. */ *csum_updated |= nfp_fl_csum_l4_to_flag(ip_proto); } else if (set_ip6_src.head.len_lw) { + nfp_action += act_size; act_size = sizeof(set_ip6_src); memcpy(nfp_action, &set_ip6_src, act_size); *a_len += act_size; /* Hardware will automatically fix TCP/UDP checksum. */ *csum_updated |= nfp_fl_csum_l4_to_flag(ip_proto); - } else if (set_tport.head.len_lw) { + } + if (set_tport.head.len_lw) { + nfp_action += act_size; act_size = sizeof(set_tport); memcpy(nfp_action, &set_tport, act_size); *a_len += act_size; diff --git a/drivers/net/ethernet/qlogic/qed/qed_int.c b/drivers/net/ethernet/qlogic/qed/qed_int.c index af3a28ec04eb..0f0aba793352 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_int.c +++ b/drivers/net/ethernet/qlogic/qed/qed_int.c @@ -228,7 +228,7 @@ static int qed_grc_attn_cb(struct qed_hwfn *p_hwfn) attn_master_to_str(GET_FIELD(tmp, QED_GRC_ATTENTION_MASTER)), GET_FIELD(tmp2, QED_GRC_ATTENTION_PF), (GET_FIELD(tmp2, QED_GRC_ATTENTION_PRIV) == - QED_GRC_ATTENTION_PRIV_VF) ? "VF" : "(Ireelevant)", + QED_GRC_ATTENTION_PRIV_VF) ? "VF" : "(Irrelevant)", GET_FIELD(tmp2, QED_GRC_ATTENTION_VF)); out: diff --git a/drivers/net/ethernet/qlogic/qla3xxx.c b/drivers/net/ethernet/qlogic/qla3xxx.c index b48f76182049..10b075bc5959 100644 --- a/drivers/net/ethernet/qlogic/qla3xxx.c +++ b/drivers/net/ethernet/qlogic/qla3xxx.c @@ -380,8 +380,6 @@ static void fm93c56a_select(struct ql3_adapter *qdev) qdev->eeprom_cmd_data = AUBURN_EEPROM_CS_1; ql_write_nvram_reg(qdev, spir, ISP_NVRAM_MASK | qdev->eeprom_cmd_data); - ql_write_nvram_reg(qdev, spir, - ((ISP_NVRAM_MASK << 16) | qdev->eeprom_cmd_data)); } /* diff --git a/drivers/net/ethernet/realtek/r8169.c b/drivers/net/ethernet/realtek/r8169.c index 3a5e6160bf0d..2c350099b83c 100644 --- a/drivers/net/ethernet/realtek/r8169.c +++ b/drivers/net/ethernet/realtek/r8169.c @@ -6549,17 +6549,15 @@ static int rtl8169_poll(struct napi_struct *napi, int budget) struct rtl8169_private *tp = container_of(napi, struct rtl8169_private, napi); struct net_device *dev = tp->dev; u16 enable_mask = RTL_EVENT_NAPI | tp->event_slow; - int work_done= 0; + int work_done; u16 status; status = rtl_get_events(tp); rtl_ack_events(tp, status & ~tp->event_slow); - if (status & RTL_EVENT_NAPI_RX) - work_done = rtl_rx(dev, tp, (u32) budget); + work_done = rtl_rx(dev, tp, (u32) budget); - if (status & RTL_EVENT_NAPI_TX) - rtl_tx(dev, tp); + rtl_tx(dev, tp); if (status & tp->event_slow) { enable_mask &= ~tp->event_slow; @@ -7093,20 +7091,12 @@ static int rtl_alloc_irq(struct rtl8169_private *tp) { unsigned int flags; - switch (tp->mac_version) { - case RTL_GIGA_MAC_VER_01 ... RTL_GIGA_MAC_VER_06: + if (tp->mac_version <= RTL_GIGA_MAC_VER_06) { RTL_W8(tp, Cfg9346, Cfg9346_Unlock); RTL_W8(tp, Config2, RTL_R8(tp, Config2) & ~MSIEnable); RTL_W8(tp, Cfg9346, Cfg9346_Lock); flags = PCI_IRQ_LEGACY; - break; - case RTL_GIGA_MAC_VER_39 ... RTL_GIGA_MAC_VER_40: - /* This version was reported to have issues with resume - * from suspend when using MSI-X - */ - flags = PCI_IRQ_LEGACY | PCI_IRQ_MSI; - break; - default: + } else { flags = PCI_IRQ_ALL_TYPES; } diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c index 6acb6b5718b9..493cd382b8aa 100644 --- a/drivers/net/geneve.c +++ b/drivers/net/geneve.c @@ -830,12 +830,8 @@ static int geneve_xmit_skb(struct sk_buff *skb, struct net_device *dev, if (IS_ERR(rt)) return PTR_ERR(rt); - if (skb_dst(skb)) { - int mtu = dst_mtu(&rt->dst) - GENEVE_IPV4_HLEN - - info->options_len; - - skb_dst_update_pmtu(skb, mtu); - } + skb_tunnel_check_pmtu(skb, &rt->dst, + GENEVE_IPV4_HLEN + info->options_len); sport = udp_flow_src_port(geneve->net, skb, 1, USHRT_MAX, true); if (geneve->collect_md) { @@ -876,11 +872,7 @@ static int geneve6_xmit_skb(struct sk_buff *skb, struct net_device *dev, if (IS_ERR(dst)) return PTR_ERR(dst); - if (skb_dst(skb)) { - int mtu = dst_mtu(dst) - GENEVE_IPV6_HLEN - info->options_len; - - skb_dst_update_pmtu(skb, mtu); - } + skb_tunnel_check_pmtu(skb, dst, GENEVE_IPV6_HLEN + info->options_len); sport = udp_flow_src_port(geneve->net, skb, 1, USHRT_MAX, true); if (geneve->collect_md) { diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index dab504ec5e50..ddfa3f24204c 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -2218,8 +2218,9 @@ static void virtnet_freeze_down(struct virtio_device *vdev) /* Make sure no work handler is accessing the device */ flush_work(&vi->config_work); + netif_tx_lock_bh(vi->dev); netif_device_detach(vi->dev); - netif_tx_disable(vi->dev); + netif_tx_unlock_bh(vi->dev); cancel_delayed_work_sync(&vi->refill); if (netif_running(vi->dev)) { @@ -2255,7 +2256,9 @@ static int virtnet_restore_up(struct virtio_device *vdev) } } + netif_tx_lock_bh(vi->dev); netif_device_attach(vi->dev); + netif_tx_unlock_bh(vi->dev); return err; } diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c index 2b8da2b7e721..27bd586b94b0 100644 --- a/drivers/net/vxlan.c +++ b/drivers/net/vxlan.c @@ -2194,11 +2194,7 @@ static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev, } ndst = &rt->dst; - if (skb_dst(skb)) { - int mtu = dst_mtu(ndst) - VXLAN_HEADROOM; - - skb_dst_update_pmtu(skb, mtu); - } + skb_tunnel_check_pmtu(skb, ndst, VXLAN_HEADROOM); tos = ip_tunnel_ecn_encap(tos, old_iph, skb); ttl = ttl ? : ip4_dst_hoplimit(&rt->dst); @@ -2235,11 +2231,7 @@ static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev, goto out_unlock; } - if (skb_dst(skb)) { - int mtu = dst_mtu(ndst) - VXLAN6_HEADROOM; - - skb_dst_update_pmtu(skb, mtu); - } + skb_tunnel_check_pmtu(skb, ndst, VXLAN6_HEADROOM); tos = ip_tunnel_ecn_encap(tos, old_iph, skb); ttl = ttl ? : ip6_dst_hoplimit(ndst); diff --git a/drivers/nvdimm/blk.c b/drivers/nvdimm/blk.c index 62e9cb167aad..db45c6bbb7bb 100644 --- a/drivers/nvdimm/blk.c +++ b/drivers/nvdimm/blk.c @@ -290,7 +290,7 @@ static int nsblk_attach_disk(struct nd_namespace_blk *nsblk) } set_capacity(disk, available_disk_size >> SECTOR_SHIFT); - device_add_disk(dev, disk); + device_add_disk(dev, disk, NULL); revalidate_disk(disk); return 0; } diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index 0360c015f658..b123b0dcf274 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c @@ -1556,7 +1556,7 @@ static int btt_blk_init(struct btt *btt) } } set_capacity(btt->btt_disk, btt->nlba * btt->sector_size >> 9); - device_add_disk(&btt->nd_btt->dev, btt->btt_disk); + device_add_disk(&btt->nd_btt->dev, btt->btt_disk, NULL); btt->nd_btt->size = btt->nlba * (u64)btt->sector_size; revalidate_disk(btt->btt_disk); diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 6071e2942053..a75d10c23d80 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -474,7 +474,7 @@ static int pmem_attach_disk(struct device *dev, gendev = disk_to_dev(disk); gendev->groups = pmem_attribute_groups; - device_add_disk(dev, disk); + device_add_disk(dev, disk, NULL); if (devm_add_action_or_reset(dev, pmem_release_disk, pmem)) return -ENOMEM; diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index dd8ec1dd9219..9e4a30b05bd2 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -971,7 +971,7 @@ static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid, uuid_copy(&ids->uuid, data + pos + sizeof(*cur)); break; default: - /* Skip unnkown types */ + /* Skip unknown types */ len = cur->nidl; break; } @@ -1132,7 +1132,7 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) return nvme_submit_user_cmd(ns->queue, &c, (void __user *)(uintptr_t)io.addr, length, - metadata, meta_len, io.slba, NULL, 0); + metadata, meta_len, lower_32_bits(io.slba), NULL, 0); } static u32 nvme_known_admin_effects(u8 opcode) @@ -2076,7 +2076,7 @@ static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ct nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE); if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) { - strncpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE); + strlcpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE); return; } @@ -2729,11 +2729,19 @@ static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj, return a->mode; } -const struct attribute_group nvme_ns_id_attr_group = { +static const struct attribute_group nvme_ns_id_attr_group = { .attrs = nvme_ns_id_attrs, .is_visible = nvme_ns_id_attrs_are_visible, }; +const struct attribute_group *nvme_ns_id_attr_groups[] = { + &nvme_ns_id_attr_group, +#ifdef CONFIG_NVM + &nvme_nvm_attr_group, +#endif + NULL, +}; + #define nvme_show_str_function(field) \ static ssize_t field##_show(struct device *dev, \ struct device_attribute *attr, char *buf) \ @@ -2900,9 +2908,14 @@ static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl, unsigned nsid, struct nvme_id_ns *id) { struct nvme_ns_head *head; + size_t size = sizeof(*head); int ret = -ENOMEM; - head = kzalloc(sizeof(*head), GFP_KERNEL); +#ifdef CONFIG_NVME_MULTIPATH + size += num_possible_nodes() * sizeof(struct nvme_ns *); +#endif + + head = kzalloc(size, GFP_KERNEL); if (!head) goto out; ret = ida_simple_get(&ctrl->subsys->ns_ida, 1, 0, GFP_KERNEL); @@ -3099,14 +3112,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid) nvme_get_ctrl(ctrl); - device_add_disk(ctrl->device, ns->disk); - if (sysfs_create_group(&disk_to_dev(ns->disk)->kobj, - &nvme_ns_id_attr_group)) - pr_warn("%s: failed to create sysfs group for identification\n", - ns->disk->disk_name); - if (ns->ndev && nvme_nvm_register_sysfs(ns)) - pr_warn("%s: failed to register lightnvm sysfs group for identification\n", - ns->disk->disk_name); + device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups); nvme_mpath_add_disk(ns, id); nvme_fault_inject_init(ns); @@ -3132,10 +3138,6 @@ static void nvme_ns_remove(struct nvme_ns *ns) nvme_fault_inject_fini(ns); if (ns->disk && ns->disk->flags & GENHD_FL_UP) { - sysfs_remove_group(&disk_to_dev(ns->disk)->kobj, - &nvme_ns_id_attr_group); - if (ns->ndev) - nvme_nvm_unregister_sysfs(ns); del_gendisk(ns->disk); blk_cleanup_queue(ns->queue); if (blk_get_integrity(ns->disk)) @@ -3143,8 +3145,8 @@ static void nvme_ns_remove(struct nvme_ns *ns) } mutex_lock(&ns->ctrl->subsys->lock); - nvme_mpath_clear_current_path(ns); list_del_rcu(&ns->siblings); + nvme_mpath_clear_current_path(ns); mutex_unlock(&ns->ctrl->subsys->lock); down_write(&ns->ctrl->namespaces_rwsem); @@ -3411,16 +3413,21 @@ static void nvme_fw_act_work(struct work_struct *work) static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result) { - switch ((result & 0xff00) >> 8) { + u32 aer_notice_type = (result & 0xff00) >> 8; + + switch (aer_notice_type) { case NVME_AER_NOTICE_NS_CHANGED: + trace_nvme_async_event(ctrl, aer_notice_type); set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events); nvme_queue_scan(ctrl); break; case NVME_AER_NOTICE_FW_ACT_STARTING: + trace_nvme_async_event(ctrl, aer_notice_type); queue_work(nvme_wq, &ctrl->fw_act_work); break; #ifdef CONFIG_NVME_MULTIPATH case NVME_AER_NOTICE_ANA: + trace_nvme_async_event(ctrl, aer_notice_type); if (!ctrl->ana_log_buf) break; queue_work(nvme_wq, &ctrl->ana_work); @@ -3435,11 +3442,12 @@ void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status, volatile union nvme_result *res) { u32 result = le32_to_cpu(res->u32); + u32 aer_type = result & 0x07; if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS) return; - switch (result & 0x7) { + switch (aer_type) { case NVME_AER_NOTICE: nvme_handle_aen_notice(ctrl, result); break; @@ -3447,6 +3455,7 @@ void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status, case NVME_AER_SMART: case NVME_AER_CSS: case NVME_AER_VS: + trace_nvme_async_event(ctrl, aer_type); ctrl->aen_result = result; break; default: diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c index 206d63cb1afc..bd0969db6225 100644 --- a/drivers/nvme/host/fabrics.c +++ b/drivers/nvme/host/fabrics.c @@ -552,8 +552,11 @@ blk_status_t nvmf_fail_nonready_command(struct nvme_ctrl *ctrl, ctrl->state != NVME_CTRL_DEAD && !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH)) return BLK_STS_RESOURCE; - nvme_req(rq)->status = NVME_SC_ABORT_REQ; - return BLK_STS_IOERR; + + nvme_req(rq)->status = NVME_SC_HOST_PATH_ERROR; + blk_mq_start_request(rq); + nvme_complete_rq(rq); + return BLK_STS_OK; } EXPORT_SYMBOL_GPL(nvmf_fail_nonready_command); @@ -865,6 +868,36 @@ static int nvmf_check_required_opts(struct nvmf_ctrl_options *opts, return 0; } +bool nvmf_ip_options_match(struct nvme_ctrl *ctrl, + struct nvmf_ctrl_options *opts) +{ + if (!nvmf_ctlr_matches_baseopts(ctrl, opts) || + strcmp(opts->traddr, ctrl->opts->traddr) || + strcmp(opts->trsvcid, ctrl->opts->trsvcid)) + return false; + + /* + * Checking the local address is rough. In most cases, none is specified + * and the host port is selected by the stack. + * + * Assume no match if: + * - local address is specified and address is not the same + * - local address is not specified but remote is, or vice versa + * (admin using specific host_traddr when it matters). + */ + if ((opts->mask & NVMF_OPT_HOST_TRADDR) && + (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)) { + if (strcmp(opts->host_traddr, ctrl->opts->host_traddr)) + return false; + } else if ((opts->mask & NVMF_OPT_HOST_TRADDR) || + (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)) { + return false; + } + + return true; +} +EXPORT_SYMBOL_GPL(nvmf_ip_options_match); + static int nvmf_check_allowed_opts(struct nvmf_ctrl_options *opts, unsigned int allowed_opts) { diff --git a/drivers/nvme/host/fabrics.h b/drivers/nvme/host/fabrics.h index aa2fdb2a2e8f..6ea6275f332a 100644 --- a/drivers/nvme/host/fabrics.h +++ b/drivers/nvme/host/fabrics.h @@ -166,6 +166,8 @@ blk_status_t nvmf_fail_nonready_command(struct nvme_ctrl *ctrl, struct request *rq); bool __nvmf_check_ready(struct nvme_ctrl *ctrl, struct request *rq, bool queue_live); +bool nvmf_ip_options_match(struct nvme_ctrl *ctrl, + struct nvmf_ctrl_options *opts); static inline bool nvmf_check_ready(struct nvme_ctrl *ctrl, struct request *rq, bool queue_live) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 611e70cae754..e52b9d3c0bd6 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -20,6 +20,7 @@ #include <uapi/scsi/fc/fc_fs.h> #include <uapi/scsi/fc/fc_els.h> #include <linux/delay.h> +#include <linux/overflow.h> #include "nvme.h" #include "fabrics.h" @@ -104,6 +105,12 @@ struct nvme_fc_fcp_op { struct nvme_fc_ersp_iu rsp_iu; }; +struct nvme_fcp_op_w_sgl { + struct nvme_fc_fcp_op op; + struct scatterlist sgl[SG_CHUNK_SIZE]; + uint8_t priv[0]; +}; + struct nvme_fc_lport { struct nvme_fc_local_port localport; @@ -122,6 +129,7 @@ struct nvme_fc_rport { struct list_head endp_list; /* for lport->endp_list */ struct list_head ctrl_list; struct list_head ls_req_list; + struct list_head disc_list; struct device *dev; /* physical device for dma */ struct nvme_fc_lport *lport; spinlock_t lock; @@ -210,7 +218,6 @@ static DEFINE_IDA(nvme_fc_ctrl_cnt); * These items are short-term. They will eventually be moved into * a generic FC class. See comments in module init. */ -static struct class *fc_class; static struct device *fc_udev_device; @@ -317,7 +324,7 @@ out_done: * @template: LLDD entrypoints and operational parameters for the port * @dev: physical hardware device node port corresponds to. Will be * used for DMA mappings - * @lport_p: pointer to a local port pointer. Upon success, the routine + * @portptr: pointer to a local port pointer. Upon success, the routine * will allocate a nvme_fc_local_port structure and place its * address in the local port pointer. Upon failure, local port * pointer will be set to 0. @@ -425,8 +432,7 @@ EXPORT_SYMBOL_GPL(nvme_fc_register_localport); * nvme_fc_unregister_localport - transport entry point called by an * LLDD to deregister/remove a previously * registered a NVME host FC port. - * @localport: pointer to the (registered) local port that is to be - * deregistered. + * @portptr: pointer to the (registered) local port that is to be deregistered. * * Returns: * a completion status. Must be 0 upon success; a negative errno @@ -507,6 +513,7 @@ nvme_fc_free_rport(struct kref *ref) list_del(&rport->endp_list); spin_unlock_irqrestore(&nvme_fc_lock, flags); + WARN_ON(!list_empty(&rport->disc_list)); ida_simple_remove(&lport->endp_cnt, rport->remoteport.port_num); kfree(rport); @@ -631,7 +638,7 @@ __nvme_fc_set_dev_loss_tmo(struct nvme_fc_rport *rport, * @localport: pointer to the (registered) local port that the remote * subsystem port is connected to. * @pinfo: pointer to information about the port to be registered - * @rport_p: pointer to a remote port pointer. Upon success, the routine + * @portptr: pointer to a remote port pointer. Upon success, the routine * will allocate a nvme_fc_remote_port structure and place its * address in the remote port pointer. Upon failure, remote port * pointer will be set to 0. @@ -694,6 +701,7 @@ nvme_fc_register_remoteport(struct nvme_fc_local_port *localport, INIT_LIST_HEAD(&newrec->endp_list); INIT_LIST_HEAD(&newrec->ctrl_list); INIT_LIST_HEAD(&newrec->ls_req_list); + INIT_LIST_HEAD(&newrec->disc_list); kref_init(&newrec->ref); atomic_set(&newrec->act_ctrl_cnt, 0); spin_lock_init(&newrec->lock); @@ -807,8 +815,8 @@ nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl) * nvme_fc_unregister_remoteport - transport entry point called by an * LLDD to deregister/remove a previously * registered a NVME subsystem FC port. - * @remoteport: pointer to the (registered) remote port that is to be - * deregistered. + * @portptr: pointer to the (registered) remote port that is to be + * deregistered. * * Returns: * a completion status. Must be 0 upon success; a negative errno @@ -1385,7 +1393,7 @@ nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status) __nvme_fc_finish_ls_req(lsop); - /* fc-nvme iniator doesn't care about success or failure of cmd */ + /* fc-nvme initiator doesn't care about success or failure of cmd */ kfree(lsop); } @@ -1685,6 +1693,8 @@ __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue, struct nvme_fc_fcp_op *op, struct request *rq, u32 rqno) { + struct nvme_fcp_op_w_sgl *op_w_sgl = + container_of(op, typeof(*op_w_sgl), op); struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu; int ret = 0; @@ -1694,7 +1704,6 @@ __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl, op->fcp_req.rspaddr = &op->rsp_iu; op->fcp_req.rsplen = sizeof(op->rsp_iu); op->fcp_req.done = nvme_fc_fcpio_done; - op->fcp_req.first_sgl = (struct scatterlist *)&op[1]; op->fcp_req.private = &op->fcp_req.first_sgl[SG_CHUNK_SIZE]; op->ctrl = ctrl; op->queue = queue; @@ -1733,12 +1742,17 @@ nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, unsigned int numa_node) { struct nvme_fc_ctrl *ctrl = set->driver_data; - struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq); + struct nvme_fcp_op_w_sgl *op = blk_mq_rq_to_pdu(rq); int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0; struct nvme_fc_queue *queue = &ctrl->queues[queue_idx]; + int res; nvme_req(rq)->ctrl = &ctrl->ctrl; - return __nvme_fc_init_request(ctrl, queue, op, rq, queue->rqcnt++); + res = __nvme_fc_init_request(ctrl, queue, &op->op, rq, queue->rqcnt++); + if (res) + return res; + op->op.fcp_req.first_sgl = &op->sgl[0]; + return res; } static int @@ -1768,7 +1782,6 @@ nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl) } aen_op->flags = FCOP_FLAGS_AEN; - aen_op->fcp_req.first_sgl = NULL; /* no sg list */ aen_op->fcp_req.private = private; memset(sqe, 0, sizeof(*sqe)); @@ -2422,10 +2435,9 @@ nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl) ctrl->tag_set.reserved_tags = 1; /* fabric connect */ ctrl->tag_set.numa_node = NUMA_NO_NODE; ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; - ctrl->tag_set.cmd_size = sizeof(struct nvme_fc_fcp_op) + - (SG_CHUNK_SIZE * - sizeof(struct scatterlist)) + - ctrl->lport->ops->fcprqst_priv_sz; + ctrl->tag_set.cmd_size = + struct_size((struct nvme_fcp_op_w_sgl *)NULL, priv, + ctrl->lport->ops->fcprqst_priv_sz); ctrl->tag_set.driver_data = ctrl; ctrl->tag_set.nr_hw_queues = ctrl->ctrl.queue_count - 1; ctrl->tag_set.timeout = NVME_IO_TIMEOUT; @@ -3027,10 +3039,9 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, ctrl->admin_tag_set.queue_depth = NVME_AQ_MQ_TAG_DEPTH; ctrl->admin_tag_set.reserved_tags = 2; /* fabric connect + Keep-Alive */ ctrl->admin_tag_set.numa_node = NUMA_NO_NODE; - ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_fc_fcp_op) + - (SG_CHUNK_SIZE * - sizeof(struct scatterlist)) + - ctrl->lport->ops->fcprqst_priv_sz; + ctrl->admin_tag_set.cmd_size = + struct_size((struct nvme_fcp_op_w_sgl *)NULL, priv, + ctrl->lport->ops->fcprqst_priv_sz); ctrl->admin_tag_set.driver_data = ctrl; ctrl->admin_tag_set.nr_hw_queues = 1; ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT; @@ -3159,7 +3170,7 @@ nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen) substring_t wwn = { name, &name[sizeof(name)-1] }; int nnoffset, pnoffset; - /* validate it string one of the 2 allowed formats */ + /* validate if string is one of the 2 allowed formats */ if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH && !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) && !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET], @@ -3254,6 +3265,90 @@ static struct nvmf_transport_ops nvme_fc_transport = { .create_ctrl = nvme_fc_create_ctrl, }; +/* Arbitrary successive failures max. With lots of subsystems could be high */ +#define DISCOVERY_MAX_FAIL 20 + +static ssize_t nvme_fc_nvme_discovery_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + unsigned long flags; + LIST_HEAD(local_disc_list); + struct nvme_fc_lport *lport; + struct nvme_fc_rport *rport; + int failcnt = 0; + + spin_lock_irqsave(&nvme_fc_lock, flags); +restart: + list_for_each_entry(lport, &nvme_fc_lport_list, port_list) { + list_for_each_entry(rport, &lport->endp_list, endp_list) { + if (!nvme_fc_lport_get(lport)) + continue; + if (!nvme_fc_rport_get(rport)) { + /* + * This is a temporary condition. Upon restart + * this rport will be gone from the list. + * + * Revert the lport put and retry. Anything + * added to the list already will be skipped (as + * they are no longer list_empty). Loops should + * resume at rports that were not yet seen. + */ + nvme_fc_lport_put(lport); + + if (failcnt++ < DISCOVERY_MAX_FAIL) + goto restart; + + pr_err("nvme_discovery: too many reference " + "failures\n"); + goto process_local_list; + } + if (list_empty(&rport->disc_list)) + list_add_tail(&rport->disc_list, + &local_disc_list); + } + } + +process_local_list: + while (!list_empty(&local_disc_list)) { + rport = list_first_entry(&local_disc_list, + struct nvme_fc_rport, disc_list); + list_del_init(&rport->disc_list); + spin_unlock_irqrestore(&nvme_fc_lock, flags); + + lport = rport->lport; + /* signal discovery. Won't hurt if it repeats */ + nvme_fc_signal_discovery_scan(lport, rport); + nvme_fc_rport_put(rport); + nvme_fc_lport_put(lport); + + spin_lock_irqsave(&nvme_fc_lock, flags); + } + spin_unlock_irqrestore(&nvme_fc_lock, flags); + + return count; +} +static DEVICE_ATTR(nvme_discovery, 0200, NULL, nvme_fc_nvme_discovery_store); + +static struct attribute *nvme_fc_attrs[] = { + &dev_attr_nvme_discovery.attr, + NULL +}; + +static struct attribute_group nvme_fc_attr_group = { + .attrs = nvme_fc_attrs, +}; + +static const struct attribute_group *nvme_fc_attr_groups[] = { + &nvme_fc_attr_group, + NULL +}; + +static struct class fc_class = { + .name = "fc", + .dev_groups = nvme_fc_attr_groups, + .owner = THIS_MODULE, +}; + static int __init nvme_fc_init_module(void) { int ret; @@ -3272,16 +3367,16 @@ static int __init nvme_fc_init_module(void) * put in place, this code will move to a more generic * location for the class. */ - fc_class = class_create(THIS_MODULE, "fc"); - if (IS_ERR(fc_class)) { + ret = class_register(&fc_class); + if (ret) { pr_err("couldn't register class fc\n"); - return PTR_ERR(fc_class); + return ret; } /* * Create a device for the FC-centric udev events */ - fc_udev_device = device_create(fc_class, NULL, MKDEV(0, 0), NULL, + fc_udev_device = device_create(&fc_class, NULL, MKDEV(0, 0), NULL, "fc_udev_device"); if (IS_ERR(fc_udev_device)) { pr_err("couldn't create fc_udev device!\n"); @@ -3296,9 +3391,9 @@ static int __init nvme_fc_init_module(void) return 0; out_destroy_device: - device_destroy(fc_class, MKDEV(0, 0)); + device_destroy(&fc_class, MKDEV(0, 0)); out_destroy_class: - class_destroy(fc_class); + class_unregister(&fc_class); return ret; } @@ -3313,8 +3408,8 @@ static void __exit nvme_fc_exit_module(void) ida_destroy(&nvme_fc_local_port_cnt); ida_destroy(&nvme_fc_ctrl_cnt); - device_destroy(fc_class, MKDEV(0, 0)); - class_destroy(fc_class); + device_destroy(&fc_class, MKDEV(0, 0)); + class_unregister(&fc_class); } module_init(nvme_fc_init_module); diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c index 6fe5923c95d4..a4f3b263cd6c 100644 --- a/drivers/nvme/host/lightnvm.c +++ b/drivers/nvme/host/lightnvm.c @@ -567,13 +567,13 @@ static int nvme_nvm_set_bb_tbl(struct nvm_dev *nvmdev, struct ppa_addr *ppas, * Expect the lba in device format */ static int nvme_nvm_get_chk_meta(struct nvm_dev *ndev, - struct nvm_chk_meta *meta, - sector_t slba, int nchks) + sector_t slba, int nchks, + struct nvm_chk_meta *meta) { struct nvm_geo *geo = &ndev->geo; struct nvme_ns *ns = ndev->q->queuedata; struct nvme_ctrl *ctrl = ns->ctrl; - struct nvme_nvm_chk_meta *dev_meta = (struct nvme_nvm_chk_meta *)meta; + struct nvme_nvm_chk_meta *dev_meta, *dev_meta_off; struct ppa_addr ppa; size_t left = nchks * sizeof(struct nvme_nvm_chk_meta); size_t log_pos, offset, len; @@ -585,6 +585,10 @@ static int nvme_nvm_get_chk_meta(struct nvm_dev *ndev, */ max_len = min_t(unsigned int, ctrl->max_hw_sectors << 9, 256 * 1024); + dev_meta = kmalloc(max_len, GFP_KERNEL); + if (!dev_meta) + return -ENOMEM; + /* Normalize lba address space to obtain log offset */ ppa.ppa = slba; ppa = dev_to_generic_addr(ndev, ppa); @@ -598,6 +602,9 @@ static int nvme_nvm_get_chk_meta(struct nvm_dev *ndev, while (left) { len = min_t(unsigned int, left, max_len); + memset(dev_meta, 0, max_len); + dev_meta_off = dev_meta; + ret = nvme_get_log(ctrl, ns->head->ns_id, NVME_NVM_LOG_REPORT_CHUNK, 0, dev_meta, len, offset); @@ -607,21 +614,23 @@ static int nvme_nvm_get_chk_meta(struct nvm_dev *ndev, } for (i = 0; i < len; i += sizeof(struct nvme_nvm_chk_meta)) { - meta->state = dev_meta->state; - meta->type = dev_meta->type; - meta->wi = dev_meta->wi; - meta->slba = le64_to_cpu(dev_meta->slba); - meta->cnlb = le64_to_cpu(dev_meta->cnlb); - meta->wp = le64_to_cpu(dev_meta->wp); + meta->state = dev_meta_off->state; + meta->type = dev_meta_off->type; + meta->wi = dev_meta_off->wi; + meta->slba = le64_to_cpu(dev_meta_off->slba); + meta->cnlb = le64_to_cpu(dev_meta_off->cnlb); + meta->wp = le64_to_cpu(dev_meta_off->wp); meta++; - dev_meta++; + dev_meta_off++; } offset += len; left -= len; } + kfree(dev_meta); + return ret; } @@ -968,6 +977,9 @@ void nvme_nvm_update_nvm_info(struct nvme_ns *ns) struct nvm_dev *ndev = ns->ndev; struct nvm_geo *geo = &ndev->geo; + if (geo->version == NVM_OCSSD_SPEC_12) + return; + geo->csecs = 1 << ns->lba_shift; geo->sos = ns->ms; } @@ -1190,10 +1202,29 @@ static NVM_DEV_ATTR_12_RO(multiplane_modes); static NVM_DEV_ATTR_12_RO(media_capabilities); static NVM_DEV_ATTR_12_RO(max_phys_secs); -static struct attribute *nvm_dev_attrs_12[] = { +/* 2.0 values */ +static NVM_DEV_ATTR_20_RO(groups); +static NVM_DEV_ATTR_20_RO(punits); +static NVM_DEV_ATTR_20_RO(chunks); +static NVM_DEV_ATTR_20_RO(clba); +static NVM_DEV_ATTR_20_RO(ws_min); +static NVM_DEV_ATTR_20_RO(ws_opt); +static NVM_DEV_ATTR_20_RO(maxoc); +static NVM_DEV_ATTR_20_RO(maxocpu); +static NVM_DEV_ATTR_20_RO(mw_cunits); +static NVM_DEV_ATTR_20_RO(write_typ); +static NVM_DEV_ATTR_20_RO(write_max); +static NVM_DEV_ATTR_20_RO(reset_typ); +static NVM_DEV_ATTR_20_RO(reset_max); + +static struct attribute *nvm_dev_attrs[] = { + /* version agnostic attrs */ &dev_attr_version.attr, &dev_attr_capabilities.attr, + &dev_attr_read_typ.attr, + &dev_attr_read_max.attr, + /* 1.2 attrs */ &dev_attr_vendor_opcode.attr, &dev_attr_device_mode.attr, &dev_attr_media_manager.attr, @@ -1208,8 +1239,6 @@ static struct attribute *nvm_dev_attrs_12[] = { &dev_attr_page_size.attr, &dev_attr_hw_sector_size.attr, &dev_attr_oob_sector_size.attr, - &dev_attr_read_typ.attr, - &dev_attr_read_max.attr, &dev_attr_prog_typ.attr, &dev_attr_prog_max.attr, &dev_attr_erase_typ.attr, @@ -1218,33 +1247,7 @@ static struct attribute *nvm_dev_attrs_12[] = { &dev_attr_media_capabilities.attr, &dev_attr_max_phys_secs.attr, - NULL, -}; - -static const struct attribute_group nvm_dev_attr_group_12 = { - .name = "lightnvm", - .attrs = nvm_dev_attrs_12, -}; - -/* 2.0 values */ -static NVM_DEV_ATTR_20_RO(groups); -static NVM_DEV_ATTR_20_RO(punits); -static NVM_DEV_ATTR_20_RO(chunks); -static NVM_DEV_ATTR_20_RO(clba); -static NVM_DEV_ATTR_20_RO(ws_min); -static NVM_DEV_ATTR_20_RO(ws_opt); -static NVM_DEV_ATTR_20_RO(maxoc); -static NVM_DEV_ATTR_20_RO(maxocpu); -static NVM_DEV_ATTR_20_RO(mw_cunits); -static NVM_DEV_ATTR_20_RO(write_typ); -static NVM_DEV_ATTR_20_RO(write_max); -static NVM_DEV_ATTR_20_RO(reset_typ); -static NVM_DEV_ATTR_20_RO(reset_max); - -static struct attribute *nvm_dev_attrs_20[] = { - &dev_attr_version.attr, - &dev_attr_capabilities.attr, - + /* 2.0 attrs */ &dev_attr_groups.attr, &dev_attr_punits.attr, &dev_attr_chunks.attr, @@ -1255,8 +1258,6 @@ static struct attribute *nvm_dev_attrs_20[] = { &dev_attr_maxocpu.attr, &dev_attr_mw_cunits.attr, - &dev_attr_read_typ.attr, - &dev_attr_read_max.attr, &dev_attr_write_typ.attr, &dev_attr_write_max.attr, &dev_attr_reset_typ.attr, @@ -1265,44 +1266,38 @@ static struct attribute *nvm_dev_attrs_20[] = { NULL, }; -static const struct attribute_group nvm_dev_attr_group_20 = { - .name = "lightnvm", - .attrs = nvm_dev_attrs_20, -}; - -int nvme_nvm_register_sysfs(struct nvme_ns *ns) +static umode_t nvm_dev_attrs_visible(struct kobject *kobj, + struct attribute *attr, int index) { + struct device *dev = container_of(kobj, struct device, kobj); + struct gendisk *disk = dev_to_disk(dev); + struct nvme_ns *ns = disk->private_data; struct nvm_dev *ndev = ns->ndev; - struct nvm_geo *geo = &ndev->geo; + struct device_attribute *dev_attr = + container_of(attr, typeof(*dev_attr), attr); if (!ndev) - return -EINVAL; - - switch (geo->major_ver_id) { - case 1: - return sysfs_create_group(&disk_to_dev(ns->disk)->kobj, - &nvm_dev_attr_group_12); - case 2: - return sysfs_create_group(&disk_to_dev(ns->disk)->kobj, - &nvm_dev_attr_group_20); - } + return 0; - return -EINVAL; -} + if (dev_attr->show == nvm_dev_attr_show) + return attr->mode; -void nvme_nvm_unregister_sysfs(struct nvme_ns *ns) -{ - struct nvm_dev *ndev = ns->ndev; - struct nvm_geo *geo = &ndev->geo; - - switch (geo->major_ver_id) { + switch (ndev->geo.major_ver_id) { case 1: - sysfs_remove_group(&disk_to_dev(ns->disk)->kobj, - &nvm_dev_attr_group_12); + if (dev_attr->show == nvm_dev_attr_show_12) + return attr->mode; break; case 2: - sysfs_remove_group(&disk_to_dev(ns->disk)->kobj, - &nvm_dev_attr_group_20); + if (dev_attr->show == nvm_dev_attr_show_20) + return attr->mode; break; } + + return 0; } + +const struct attribute_group nvme_nvm_attr_group = { + .name = "lightnvm", + .attrs = nvm_dev_attrs, + .is_visible = nvm_dev_attrs_visible, +}; diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 9fe3fff818b8..5e3cc8c59a39 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -77,6 +77,13 @@ void nvme_failover_req(struct request *req) queue_work(nvme_wq, &ns->ctrl->ana_work); } break; + case NVME_SC_HOST_PATH_ERROR: + /* + * Temporary transport disruption in talking to the controller. + * Try to send on a new path. + */ + nvme_mpath_clear_current_path(ns); + break; default: /* * Reset the controller for any non-ANA error as we don't know @@ -110,29 +117,55 @@ static const char *nvme_ana_state_names[] = { [NVME_ANA_CHANGE] = "change", }; -static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head) +void nvme_mpath_clear_current_path(struct nvme_ns *ns) +{ + struct nvme_ns_head *head = ns->head; + int node; + + if (!head) + return; + + for_each_node(node) { + if (ns == rcu_access_pointer(head->current_path[node])) + rcu_assign_pointer(head->current_path[node], NULL); + } +} + +static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) { - struct nvme_ns *ns, *fallback = NULL; + int found_distance = INT_MAX, fallback_distance = INT_MAX, distance; + struct nvme_ns *found = NULL, *fallback = NULL, *ns; list_for_each_entry_rcu(ns, &head->list, siblings) { if (ns->ctrl->state != NVME_CTRL_LIVE || test_bit(NVME_NS_ANA_PENDING, &ns->flags)) continue; + + distance = node_distance(node, dev_to_node(ns->ctrl->dev)); + switch (ns->ana_state) { case NVME_ANA_OPTIMIZED: - rcu_assign_pointer(head->current_path, ns); - return ns; + if (distance < found_distance) { + found_distance = distance; + found = ns; + } + break; case NVME_ANA_NONOPTIMIZED: - fallback = ns; + if (distance < fallback_distance) { + fallback_distance = distance; + fallback = ns; + } break; default: break; } } - if (fallback) - rcu_assign_pointer(head->current_path, fallback); - return fallback; + if (!found) + found = fallback; + if (found) + rcu_assign_pointer(head->current_path[node], found); + return found; } static inline bool nvme_path_is_optimized(struct nvme_ns *ns) @@ -143,10 +176,12 @@ static inline bool nvme_path_is_optimized(struct nvme_ns *ns) inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head) { - struct nvme_ns *ns = srcu_dereference(head->current_path, &head->srcu); + int node = numa_node_id(); + struct nvme_ns *ns; + ns = srcu_dereference(head->current_path[node], &head->srcu); if (unlikely(!ns || !nvme_path_is_optimized(ns))) - ns = __nvme_find_path(head); + ns = __nvme_find_path(head, node); return ns; } @@ -193,7 +228,7 @@ static bool nvme_ns_head_poll(struct request_queue *q, blk_qc_t qc) int srcu_idx; srcu_idx = srcu_read_lock(&head->srcu); - ns = srcu_dereference(head->current_path, &head->srcu); + ns = srcu_dereference(head->current_path[numa_node_id()], &head->srcu); if (likely(ns && nvme_path_is_optimized(ns))) found = ns->queue->poll_fn(q, qc); srcu_read_unlock(&head->srcu, srcu_idx); @@ -282,12 +317,17 @@ static void nvme_mpath_set_live(struct nvme_ns *ns) if (!head->disk) return; - if (!(head->disk->flags & GENHD_FL_UP)) { - device_add_disk(&head->subsys->dev, head->disk); - if (sysfs_create_group(&disk_to_dev(head->disk)->kobj, - &nvme_ns_id_attr_group)) - dev_warn(&head->subsys->dev, - "failed to create id group.\n"); + if (!(head->disk->flags & GENHD_FL_UP)) + device_add_disk(&head->subsys->dev, head->disk, + nvme_ns_id_attr_groups); + + if (nvme_path_is_optimized(ns)) { + int node, srcu_idx; + + srcu_idx = srcu_read_lock(&head->srcu); + for_each_node(node) + __nvme_find_path(head, node); + srcu_read_unlock(&head->srcu, srcu_idx); } kblockd_schedule_work(&ns->head->requeue_work); @@ -494,11 +534,8 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head) { if (!head->disk) return; - if (head->disk->flags & GENHD_FL_UP) { - sysfs_remove_group(&disk_to_dev(head->disk)->kobj, - &nvme_ns_id_attr_group); + if (head->disk->flags & GENHD_FL_UP) del_gendisk(head->disk); - } blk_set_queue_dying(head->disk->queue); /* make sure all pending bios are cleaned up */ kblockd_schedule_work(&head->requeue_work); diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index bb4a2003c097..9fefba039d1e 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -277,14 +277,6 @@ struct nvme_ns_ids { * only ever has a single entry for private namespaces. */ struct nvme_ns_head { -#ifdef CONFIG_NVME_MULTIPATH - struct gendisk *disk; - struct nvme_ns __rcu *current_path; - struct bio_list requeue_list; - spinlock_t requeue_lock; - struct work_struct requeue_work; - struct mutex lock; -#endif struct list_head list; struct srcu_struct srcu; struct nvme_subsystem *subsys; @@ -293,6 +285,14 @@ struct nvme_ns_head { struct list_head entry; struct kref ref; int instance; +#ifdef CONFIG_NVME_MULTIPATH + struct gendisk *disk; + struct bio_list requeue_list; + spinlock_t requeue_lock; + struct work_struct requeue_work; + struct mutex lock; + struct nvme_ns __rcu *current_path[]; +#endif }; #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS @@ -459,7 +459,7 @@ int nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl); int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, void *log, size_t size, u64 offset); -extern const struct attribute_group nvme_ns_id_attr_group; +extern const struct attribute_group *nvme_ns_id_attr_groups[]; extern const struct block_device_operations nvme_ns_head_ops; #ifdef CONFIG_NVME_MULTIPATH @@ -474,14 +474,7 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head); int nvme_mpath_init(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id); void nvme_mpath_uninit(struct nvme_ctrl *ctrl); void nvme_mpath_stop(struct nvme_ctrl *ctrl); - -static inline void nvme_mpath_clear_current_path(struct nvme_ns *ns) -{ - struct nvme_ns_head *head = ns->head; - - if (head && ns == rcu_access_pointer(head->current_path)) - rcu_assign_pointer(head->current_path, NULL); -} +void nvme_mpath_clear_current_path(struct nvme_ns *ns); struct nvme_ns *nvme_find_path(struct nvme_ns_head *head); static inline void nvme_mpath_check_last_path(struct nvme_ns *ns) @@ -551,8 +544,7 @@ static inline void nvme_mpath_stop(struct nvme_ctrl *ctrl) void nvme_nvm_update_nvm_info(struct nvme_ns *ns); int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node); void nvme_nvm_unregister(struct nvme_ns *ns); -int nvme_nvm_register_sysfs(struct nvme_ns *ns); -void nvme_nvm_unregister_sysfs(struct nvme_ns *ns); +extern const struct attribute_group nvme_nvm_attr_group; int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd, unsigned long arg); #else static inline void nvme_nvm_update_nvm_info(struct nvme_ns *ns) {}; @@ -563,11 +555,6 @@ static inline int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, } static inline void nvme_nvm_unregister(struct nvme_ns *ns) {}; -static inline int nvme_nvm_register_sysfs(struct nvme_ns *ns) -{ - return 0; -} -static inline void nvme_nvm_unregister_sysfs(struct nvme_ns *ns) {}; static inline int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd, unsigned long arg) { diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index d668682f91df..4e023cd007e1 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -772,10 +772,10 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, if (!dma_map_sg(dev->dev, &iod->meta_sg, 1, dma_dir)) goto out_unmap; - } - if (blk_integrity_rq(req)) cmnd->rw.metadata = cpu_to_le64(sg_dma_address(&iod->meta_sg)); + } + return BLK_STS_OK; out_unmap: @@ -1249,7 +1249,7 @@ static void nvme_free_queues(struct nvme_dev *dev, int lowest) /** * nvme_suspend_queue - put queue into suspended state - * @nvmeq - queue to suspend + * @nvmeq: queue to suspend */ static int nvme_suspend_queue(struct nvme_queue *nvmeq) { @@ -2564,13 +2564,12 @@ static void nvme_remove(struct pci_dev *pdev) struct nvme_dev *dev = pci_get_drvdata(pdev); nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); - - cancel_work_sync(&dev->ctrl.reset_work); pci_set_drvdata(pdev, NULL); if (!pci_device_is_present(pdev)) { nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD); nvme_dev_disable(dev, true); + nvme_dev_remove_admin(dev); } flush_work(&dev->ctrl.reset_work); diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index dc042017c293..d181cafedc58 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -233,8 +233,15 @@ static void nvme_rdma_qp_event(struct ib_event *event, void *context) static int nvme_rdma_wait_for_cm(struct nvme_rdma_queue *queue) { - wait_for_completion_interruptible_timeout(&queue->cm_done, + int ret; + + ret = wait_for_completion_interruptible_timeout(&queue->cm_done, msecs_to_jiffies(NVME_RDMA_CONNECT_TIMEOUT_MS) + 1); + if (ret < 0) + return ret; + if (ret == 0) + return -ETIMEDOUT; + WARN_ON_ONCE(queue->cm_error > 0); return queue->cm_error; } @@ -1849,54 +1856,6 @@ static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = { .stop_ctrl = nvme_rdma_stop_ctrl, }; -static inline bool -__nvme_rdma_options_match(struct nvme_rdma_ctrl *ctrl, - struct nvmf_ctrl_options *opts) -{ - char *stdport = __stringify(NVME_RDMA_IP_PORT); - - - if (!nvmf_ctlr_matches_baseopts(&ctrl->ctrl, opts) || - strcmp(opts->traddr, ctrl->ctrl.opts->traddr)) - return false; - - if (opts->mask & NVMF_OPT_TRSVCID && - ctrl->ctrl.opts->mask & NVMF_OPT_TRSVCID) { - if (strcmp(opts->trsvcid, ctrl->ctrl.opts->trsvcid)) - return false; - } else if (opts->mask & NVMF_OPT_TRSVCID) { - if (strcmp(opts->trsvcid, stdport)) - return false; - } else if (ctrl->ctrl.opts->mask & NVMF_OPT_TRSVCID) { - if (strcmp(stdport, ctrl->ctrl.opts->trsvcid)) - return false; - } - /* else, it's a match as both have stdport. Fall to next checks */ - - /* - * checking the local address is rough. In most cases, one - * is not specified and the host port is selected by the stack. - * - * Assume no match if: - * local address is specified and address is not the same - * local address is not specified but remote is, or vice versa - * (admin using specific host_traddr when it matters). - */ - if (opts->mask & NVMF_OPT_HOST_TRADDR && - ctrl->ctrl.opts->mask & NVMF_OPT_HOST_TRADDR) { - if (strcmp(opts->host_traddr, ctrl->ctrl.opts->host_traddr)) - return false; - } else if (opts->mask & NVMF_OPT_HOST_TRADDR || - ctrl->ctrl.opts->mask & NVMF_OPT_HOST_TRADDR) - return false; - /* - * if neither controller had an host port specified, assume it's - * a match as everything else matched. - */ - - return true; -} - /* * Fails a connection request if it matches an existing controller * (association) with the same tuple: @@ -1917,7 +1876,7 @@ nvme_rdma_existing_controller(struct nvmf_ctrl_options *opts) mutex_lock(&nvme_rdma_ctrl_mutex); list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list) { - found = __nvme_rdma_options_match(ctrl, opts); + found = nvmf_ip_options_match(&ctrl->ctrl, opts); if (found) break; } @@ -1932,7 +1891,6 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev, struct nvme_rdma_ctrl *ctrl; int ret; bool changed; - char *port; ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); if (!ctrl) @@ -1940,15 +1898,21 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev, ctrl->ctrl.opts = opts; INIT_LIST_HEAD(&ctrl->list); - if (opts->mask & NVMF_OPT_TRSVCID) - port = opts->trsvcid; - else - port = __stringify(NVME_RDMA_IP_PORT); + if (!(opts->mask & NVMF_OPT_TRSVCID)) { + opts->trsvcid = + kstrdup(__stringify(NVME_RDMA_IP_PORT), GFP_KERNEL); + if (!opts->trsvcid) { + ret = -ENOMEM; + goto out_free_ctrl; + } + opts->mask |= NVMF_OPT_TRSVCID; + } ret = inet_pton_with_scope(&init_net, AF_UNSPEC, - opts->traddr, port, &ctrl->addr); + opts->traddr, opts->trsvcid, &ctrl->addr); if (ret) { - pr_err("malformed address passed: %s:%s\n", opts->traddr, port); + pr_err("malformed address passed: %s:%s\n", + opts->traddr, opts->trsvcid); goto out_free_ctrl; } diff --git a/drivers/nvme/host/trace.h b/drivers/nvme/host/trace.h index a490790d6691..196d5bd56718 100644 --- a/drivers/nvme/host/trace.h +++ b/drivers/nvme/host/trace.h @@ -156,6 +156,34 @@ TRACE_EVENT(nvme_complete_rq, ); +#define aer_name(aer) { aer, #aer } + +TRACE_EVENT(nvme_async_event, + TP_PROTO(struct nvme_ctrl *ctrl, u32 result), + TP_ARGS(ctrl, result), + TP_STRUCT__entry( + __field(int, ctrl_id) + __field(u32, result) + ), + TP_fast_assign( + __entry->ctrl_id = ctrl->instance; + __entry->result = result; + ), + TP_printk("nvme%d: NVME_AEN=%#08x [%s]", + __entry->ctrl_id, __entry->result, + __print_symbolic(__entry->result, + aer_name(NVME_AER_NOTICE_NS_CHANGED), + aer_name(NVME_AER_NOTICE_ANA), + aer_name(NVME_AER_NOTICE_FW_ACT_STARTING), + aer_name(NVME_AER_ERROR), + aer_name(NVME_AER_SMART), + aer_name(NVME_AER_CSS), + aer_name(NVME_AER_VS)) + ) +); + +#undef aer_name + #endif /* _TRACE_NVME_H */ #undef TRACE_INCLUDE_PATH diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 2008fa62a373..1179f6314323 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -58,7 +58,7 @@ static u16 nvmet_get_smart_log_nsid(struct nvmet_req *req, ns = nvmet_find_namespace(req->sq->ctrl, req->cmd->get_log_page.nsid); if (!ns) { - pr_err("nvmet : Could not find namespace id : %d\n", + pr_err("Could not find namespace id : %d\n", le32_to_cpu(req->cmd->get_log_page.nsid)); return NVME_SC_INVALID_NS; } @@ -353,7 +353,7 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req) if (req->port->inline_data_size) id->sgls |= cpu_to_le32(1 << 20); - strcpy(id->subnqn, ctrl->subsys->subsysnqn); + strlcpy(id->subnqn, ctrl->subsys->subsysnqn, sizeof(id->subnqn)); /* Max command capsule size is sqe + single page of in-capsule data */ id->ioccsz = cpu_to_le32((sizeof(struct nvme_command) + diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index b5ec96abd048..0acdff9e6842 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -1105,8 +1105,7 @@ static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port, if (!port) return NULL; - if (!strncmp(NVME_DISC_SUBSYS_NAME, subsysnqn, - NVMF_NQN_SIZE)) { + if (!strcmp(NVME_DISC_SUBSYS_NAME, subsysnqn)) { if (!kref_get_unless_zero(&nvmet_disc_subsys->ref)) return NULL; return nvmet_disc_subsys; diff --git a/drivers/nvme/target/discovery.c b/drivers/nvme/target/discovery.c index eae29f493a07..bc0aa0bf1543 100644 --- a/drivers/nvme/target/discovery.c +++ b/drivers/nvme/target/discovery.c @@ -174,7 +174,7 @@ static void nvmet_execute_identify_disc_ctrl(struct nvmet_req *req) if (req->port->inline_data_size) id->sgls |= cpu_to_le32(1 << 20); - strcpy(id->subnqn, ctrl->subsys->subsysnqn); + strlcpy(id->subnqn, ctrl->subsys->subsysnqn, sizeof(id->subnqn)); status = nvmet_copy_to_sgl(req, 0, id, sizeof(*id)); @@ -219,12 +219,10 @@ u16 nvmet_parse_discovery_cmd(struct nvmet_req *req) return NVME_SC_INVALID_OPCODE | NVME_SC_DNR; } default: - pr_err("unsupported cmd %d\n", cmd->common.opcode); + pr_err("unhandled cmd %d\n", cmd->common.opcode); return NVME_SC_INVALID_OPCODE | NVME_SC_DNR; } - pr_err("unhandled cmd %d\n", cmd->common.opcode); - return NVME_SC_INVALID_OPCODE | NVME_SC_DNR; } int __init nvmet_init_discovery(void) diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index 29b4b236afd8..409081a03b24 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -110,11 +110,19 @@ struct nvmet_fc_tgtport { struct list_head ls_busylist; struct list_head assoc_list; struct ida assoc_cnt; - struct nvmet_port *port; + struct nvmet_fc_port_entry *pe; struct kref ref; u32 max_sg_cnt; }; +struct nvmet_fc_port_entry { + struct nvmet_fc_tgtport *tgtport; + struct nvmet_port *port; + u64 node_name; + u64 port_name; + struct list_head pe_list; +}; + struct nvmet_fc_defer_fcp_req { struct list_head req_list; struct nvmefc_tgt_fcp_req *fcp_req; @@ -132,7 +140,6 @@ struct nvmet_fc_tgt_queue { atomic_t zrspcnt; atomic_t rsn; spinlock_t qlock; - struct nvmet_port *port; struct nvmet_cq nvme_cq; struct nvmet_sq nvme_sq; struct nvmet_fc_tgt_assoc *assoc; @@ -221,6 +228,7 @@ static DEFINE_SPINLOCK(nvmet_fc_tgtlock); static LIST_HEAD(nvmet_fc_target_list); static DEFINE_IDA(nvmet_fc_tgtport_cnt); +static LIST_HEAD(nvmet_fc_portentry_list); static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work); @@ -645,7 +653,6 @@ nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc, queue->qid = qid; queue->sqsize = sqsize; queue->assoc = assoc; - queue->port = assoc->tgtport->port; queue->cpu = nvmet_fc_queue_to_cpu(assoc->tgtport, qid); INIT_LIST_HEAD(&queue->fod_list); INIT_LIST_HEAD(&queue->avail_defer_list); @@ -957,6 +964,83 @@ nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport, return ret; } +static void +nvmet_fc_portentry_bind(struct nvmet_fc_tgtport *tgtport, + struct nvmet_fc_port_entry *pe, + struct nvmet_port *port) +{ + lockdep_assert_held(&nvmet_fc_tgtlock); + + pe->tgtport = tgtport; + tgtport->pe = pe; + + pe->port = port; + port->priv = pe; + + pe->node_name = tgtport->fc_target_port.node_name; + pe->port_name = tgtport->fc_target_port.port_name; + INIT_LIST_HEAD(&pe->pe_list); + + list_add_tail(&pe->pe_list, &nvmet_fc_portentry_list); +} + +static void +nvmet_fc_portentry_unbind(struct nvmet_fc_port_entry *pe) +{ + unsigned long flags; + + spin_lock_irqsave(&nvmet_fc_tgtlock, flags); + if (pe->tgtport) + pe->tgtport->pe = NULL; + list_del(&pe->pe_list); + spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); +} + +/* + * called when a targetport deregisters. Breaks the relationship + * with the nvmet port, but leaves the port_entry in place so that + * re-registration can resume operation. + */ +static void +nvmet_fc_portentry_unbind_tgt(struct nvmet_fc_tgtport *tgtport) +{ + struct nvmet_fc_port_entry *pe; + unsigned long flags; + + spin_lock_irqsave(&nvmet_fc_tgtlock, flags); + pe = tgtport->pe; + if (pe) + pe->tgtport = NULL; + tgtport->pe = NULL; + spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); +} + +/* + * called when a new targetport is registered. Looks in the + * existing nvmet port_entries to see if the nvmet layer is + * configured for the targetport's wwn's. (the targetport existed, + * nvmet configured, the lldd unregistered the tgtport, and is now + * reregistering the same targetport). If so, set the nvmet port + * port entry on the targetport. + */ +static void +nvmet_fc_portentry_rebind_tgt(struct nvmet_fc_tgtport *tgtport) +{ + struct nvmet_fc_port_entry *pe; + unsigned long flags; + + spin_lock_irqsave(&nvmet_fc_tgtlock, flags); + list_for_each_entry(pe, &nvmet_fc_portentry_list, pe_list) { + if (tgtport->fc_target_port.node_name == pe->node_name && + tgtport->fc_target_port.port_name == pe->port_name) { + WARN_ON(pe->tgtport); + tgtport->pe = pe; + pe->tgtport = tgtport; + break; + } + } + spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); +} /** * nvme_fc_register_targetport - transport entry point called by an @@ -1034,6 +1118,8 @@ nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo, goto out_free_newrec; } + nvmet_fc_portentry_rebind_tgt(newrec); + spin_lock_irqsave(&nvmet_fc_tgtlock, flags); list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list); spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); @@ -1159,8 +1245,8 @@ nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl) * nvme_fc_unregister_targetport - transport entry point called by an * LLDD to deregister/remove a previously * registered a local NVME subsystem FC port. - * @tgtport: pointer to the (registered) target port that is to be - * deregistered. + * @target_port: pointer to the (registered) target port that is to be + * deregistered. * * Returns: * a completion status. Must be 0 upon success; a negative errno @@ -1171,6 +1257,8 @@ nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port) { struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); + nvmet_fc_portentry_unbind_tgt(tgtport); + /* terminate any outstanding associations */ __nvmet_fc_free_assocs(tgtport); @@ -1661,7 +1749,7 @@ nvmet_fc_handle_ls_rqst_work(struct work_struct *work) * * If this routine returns error, the LLDD should abort the exchange. * - * @tgtport: pointer to the (registered) target port the LS was + * @target_port: pointer to the (registered) target port the LS was * received on. * @lsreq: pointer to a lsreq request structure to be used to reference * the exchange corresponding to the LS. @@ -2147,7 +2235,7 @@ nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req) /* - * Actual processing routine for received FC-NVME LS Requests from the LLD + * Actual processing routine for received FC-NVME I/O Requests from the LLD */ static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, @@ -2158,6 +2246,13 @@ nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, int ret; /* + * if there is no nvmet mapping to the targetport there + * shouldn't be requests. just terminate them. + */ + if (!tgtport->pe) + goto transport_error; + + /* * Fused commands are currently not supported in the linux * implementation. * @@ -2184,7 +2279,7 @@ nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, fod->req.cmd = &fod->cmdiubuf.sqe; fod->req.rsp = &fod->rspiubuf.cqe; - fod->req.port = fod->queue->port; + fod->req.port = tgtport->pe->port; /* clear any response payload */ memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf)); @@ -2468,7 +2563,7 @@ nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen) substring_t wwn = { name, &name[sizeof(name)-1] }; int nnoffset, pnoffset; - /* validate it string one of the 2 allowed formats */ + /* validate if string is one of the 2 allowed formats */ if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH && !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) && !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET], @@ -2508,6 +2603,7 @@ static int nvmet_fc_add_port(struct nvmet_port *port) { struct nvmet_fc_tgtport *tgtport; + struct nvmet_fc_port_entry *pe; struct nvmet_fc_traddr traddr = { 0L, 0L }; unsigned long flags; int ret; @@ -2524,24 +2620,40 @@ nvmet_fc_add_port(struct nvmet_port *port) if (ret) return ret; + pe = kzalloc(sizeof(*pe), GFP_KERNEL); + if (!pe) + return -ENOMEM; + ret = -ENXIO; spin_lock_irqsave(&nvmet_fc_tgtlock, flags); list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) { if ((tgtport->fc_target_port.node_name == traddr.nn) && (tgtport->fc_target_port.port_name == traddr.pn)) { - tgtport->port = port; - ret = 0; + /* a FC port can only be 1 nvmet port id */ + if (!tgtport->pe) { + nvmet_fc_portentry_bind(tgtport, pe, port); + ret = 0; + } else + ret = -EALREADY; break; } } spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); + + if (ret) + kfree(pe); + return ret; } static void nvmet_fc_remove_port(struct nvmet_port *port) { - /* nothing to do */ + struct nvmet_fc_port_entry *pe = port->priv; + + nvmet_fc_portentry_unbind(pe); + + kfree(pe); } static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = { diff --git a/drivers/nvme/target/fcloop.c b/drivers/nvme/target/fcloop.c index 5251689a1d9a..291f4121f516 100644 --- a/drivers/nvme/target/fcloop.c +++ b/drivers/nvme/target/fcloop.c @@ -648,6 +648,7 @@ fcloop_fcp_op(struct nvmet_fc_target_port *tgtport, break; /* Fall-Thru to RSP handling */ + /* FALLTHRU */ case NVMET_FCOP_RSP: if (fcpreq) { diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index 7bc9f6240432..f93fb5711142 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -58,7 +58,7 @@ static void nvmet_bio_done(struct bio *bio) static void nvmet_bdev_execute_rw(struct nvmet_req *req) { int sg_cnt = req->sg_cnt; - struct bio *bio = &req->b.inline_bio; + struct bio *bio; struct scatterlist *sg; sector_t sector; blk_qc_t cookie; @@ -81,7 +81,12 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) sector = le64_to_cpu(req->cmd->rw.slba); sector <<= (req->ns->blksize_shift - 9); - bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec)); + if (req->data_len <= NVMET_MAX_INLINE_DATA_LEN) { + bio = &req->b.inline_bio; + bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec)); + } else { + bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES)); + } bio_set_dev(bio, req->ns->bdev); bio->bi_iter.bi_sector = sector; bio->bi_private = req; diff --git a/drivers/nvme/target/io-cmd-file.c b/drivers/nvme/target/io-cmd-file.c index 81a9dc5290a8..39d972e2595f 100644 --- a/drivers/nvme/target/io-cmd-file.c +++ b/drivers/nvme/target/io-cmd-file.c @@ -246,7 +246,8 @@ static void nvmet_file_execute_discard(struct nvmet_req *req) break; offset = le64_to_cpu(range.slba) << req->ns->blksize_shift; - len = le32_to_cpu(range.nlb) << req->ns->blksize_shift; + len = le32_to_cpu(range.nlb); + len <<= req->ns->blksize_shift; if (offset + len > req->ns->size) { ret = NVME_SC_LBA_RANGE | NVME_SC_DNR; break; diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index ec9af4ee03b6..08f7b57a1203 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -264,6 +264,7 @@ struct nvmet_fabrics_ops { }; #define NVMET_MAX_INLINE_BIOVEC 8 +#define NVMET_MAX_INLINE_DATA_LEN NVMET_MAX_INLINE_BIOVEC * PAGE_SIZE struct nvmet_req { struct nvme_command *cmd; diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index bfc4da660bb4..bd265aceb90c 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -122,6 +122,7 @@ struct nvmet_rdma_device { int inline_page_count; }; +static struct workqueue_struct *nvmet_rdma_delete_wq; static bool nvmet_rdma_use_srq; module_param_named(use_srq, nvmet_rdma_use_srq, bool, 0444); MODULE_PARM_DESC(use_srq, "Use shared receive queue."); @@ -1267,12 +1268,12 @@ static int nvmet_rdma_queue_connect(struct rdma_cm_id *cm_id, if (queue->host_qid == 0) { /* Let inflight controller teardown complete */ - flush_scheduled_work(); + flush_workqueue(nvmet_rdma_delete_wq); } ret = nvmet_rdma_cm_accept(cm_id, queue, &event->param.conn); if (ret) { - schedule_work(&queue->release_work); + queue_work(nvmet_rdma_delete_wq, &queue->release_work); /* Destroying rdma_cm id is not needed here */ return 0; } @@ -1337,7 +1338,7 @@ static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue) if (disconnect) { rdma_disconnect(queue->cm_id); - schedule_work(&queue->release_work); + queue_work(nvmet_rdma_delete_wq, &queue->release_work); } } @@ -1367,7 +1368,7 @@ static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id *cm_id, mutex_unlock(&nvmet_rdma_queue_mutex); pr_err("failed to connect queue %d\n", queue->idx); - schedule_work(&queue->release_work); + queue_work(nvmet_rdma_delete_wq, &queue->release_work); } /** @@ -1649,8 +1650,17 @@ static int __init nvmet_rdma_init(void) if (ret) goto err_ib_client; + nvmet_rdma_delete_wq = alloc_workqueue("nvmet-rdma-delete-wq", + WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0); + if (!nvmet_rdma_delete_wq) { + ret = -ENOMEM; + goto err_unreg_transport; + } + return 0; +err_unreg_transport: + nvmet_unregister_transport(&nvmet_rdma_ops); err_ib_client: ib_unregister_client(&nvmet_rdma_ib_client); return ret; @@ -1658,6 +1668,7 @@ err_ib_client: static void __exit nvmet_rdma_exit(void) { + destroy_workqueue(nvmet_rdma_delete_wq); nvmet_unregister_transport(&nvmet_rdma_ops); ib_unregister_client(&nvmet_rdma_ib_client); WARN_ON_ONCE(!list_empty(&nvmet_rdma_queue_list)); diff --git a/drivers/of/device.c b/drivers/of/device.c index 5957cd4fa262..c7fa5a9697c9 100644 --- a/drivers/of/device.c +++ b/drivers/of/device.c @@ -170,18 +170,6 @@ int of_dma_configure(struct device *dev, struct device_node *np, bool force_dma) } EXPORT_SYMBOL_GPL(of_dma_configure); -/** - * of_dma_deconfigure - Clean up DMA configuration - * @dev: Device for which to clean up DMA configuration - * - * Clean up all configuration performed by of_dma_configure_ops() and free all - * resources that have been allocated. - */ -void of_dma_deconfigure(struct device *dev) -{ - arch_teardown_dma_ops(dev); -} - int of_device_register(struct platform_device *pdev) { device_initialize(&pdev->dev); diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c index fd2dbd7eed7b..f31ed62d518c 100644 --- a/drivers/pci/controller/vmd.c +++ b/drivers/pci/controller/vmd.c @@ -404,12 +404,10 @@ static int vmd_dma_supported(struct device *dev, u64 mask) return vmd_dma_ops(dev)->dma_supported(to_vmd_dev(dev), mask); } -#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK static u64 vmd_get_required_mask(struct device *dev) { return vmd_dma_ops(dev)->get_required_mask(to_vmd_dev(dev)); } -#endif static void vmd_teardown_dma_ops(struct vmd_dev *vmd) { @@ -450,9 +448,7 @@ static void vmd_setup_dma_ops(struct vmd_dev *vmd) ASSIGN_VMD_DMA_OPS(source, dest, sync_sg_for_device); ASSIGN_VMD_DMA_OPS(source, dest, mapping_error); ASSIGN_VMD_DMA_OPS(source, dest, dma_supported); -#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK ASSIGN_VMD_DMA_OPS(source, dest, get_required_mask); -#endif add_dma_domain(domain); } #undef ASSIGN_VMD_DMA_OPS diff --git a/drivers/perf/arm_pmu_platform.c b/drivers/perf/arm_pmu_platform.c index 96075cecb0ae..933bd8410fc2 100644 --- a/drivers/perf/arm_pmu_platform.c +++ b/drivers/perf/arm_pmu_platform.c @@ -77,14 +77,14 @@ static int pmu_parse_irq_affinity(struct device_node *node, int i) dn = of_parse_phandle(node, "interrupt-affinity", i); if (!dn) { - pr_warn("failed to parse interrupt-affinity[%d] for %s\n", - i, node->name); + pr_warn("failed to parse interrupt-affinity[%d] for %pOFn\n", + i, node); return -EINVAL; } cpu = of_cpu_node_to_id(dn); if (cpu < 0) { - pr_warn("failed to find logical CPU for %s\n", dn->name); + pr_warn("failed to find logical CPU for %pOFn\n", dn); cpu = nr_cpu_ids; } diff --git a/drivers/ptp/ptp_chardev.c b/drivers/ptp/ptp_chardev.c index 01b0e2bb3319..2012551d93e0 100644 --- a/drivers/ptp/ptp_chardev.c +++ b/drivers/ptp/ptp_chardev.c @@ -24,6 +24,8 @@ #include <linux/slab.h> #include <linux/timekeeping.h> +#include <linux/nospec.h> + #include "ptp_private.h" static int ptp_disable_pinfunc(struct ptp_clock_info *ops, @@ -248,6 +250,7 @@ long ptp_ioctl(struct posix_clock *pc, unsigned int cmd, unsigned long arg) err = -EINVAL; break; } + pin_index = array_index_nospec(pin_index, ops->n_pins); if (mutex_lock_interruptible(&ptp->pincfg_mux)) return -ERESTARTSYS; pd = ops->pin_config[pin_index]; @@ -266,6 +269,7 @@ long ptp_ioctl(struct posix_clock *pc, unsigned int cmd, unsigned long arg) err = -EINVAL; break; } + pin_index = array_index_nospec(pin_index, ops->n_pins); if (mutex_lock_interruptible(&ptp->pincfg_mux)) return -ERESTARTSYS; err = ptp_set_pinfunc(ptp, pin_index, pd.func, pd.chan); diff --git a/drivers/regulator/ltc3589.c b/drivers/regulator/ltc3589.c index 18d5b01ddcb2..63f724f260ef 100644 --- a/drivers/regulator/ltc3589.c +++ b/drivers/regulator/ltc3589.c @@ -404,7 +404,8 @@ static const struct regmap_config ltc3589_regmap_config = { .max_register = LTC3589_L2DTV2, .reg_defaults = ltc3589_reg_defaults, .num_reg_defaults = ARRAY_SIZE(ltc3589_reg_defaults), - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, .cache_type = REGCACHE_RBTREE, }; diff --git a/drivers/regulator/ltc3676.c b/drivers/regulator/ltc3676.c index 9dec1609ff66..71fd0f2a4b76 100644 --- a/drivers/regulator/ltc3676.c +++ b/drivers/regulator/ltc3676.c @@ -321,7 +321,8 @@ static const struct regmap_config ltc3676_regmap_config = { .readable_reg = ltc3676_readable_reg, .volatile_reg = ltc3676_volatile_reg, .max_register = LTC3676_CLIRQ, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, .cache_type = REGCACHE_RBTREE, }; diff --git a/drivers/s390/block/dasd_genhd.c b/drivers/s390/block/dasd_genhd.c index 7036a6c6f86f..5542d9eadfe0 100644 --- a/drivers/s390/block/dasd_genhd.c +++ b/drivers/s390/block/dasd_genhd.c @@ -76,7 +76,7 @@ int dasd_gendisk_alloc(struct dasd_block *block) gdp->queue = block->request_queue; block->gdp = gdp; set_capacity(block->gdp, 0); - device_add_disk(&base->cdev->dev, block->gdp); + device_add_disk(&base->cdev->dev, block->gdp, NULL); return 0; } diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c index 23e526cda5c1..4e8aedd50cb0 100644 --- a/drivers/s390/block/dcssblk.c +++ b/drivers/s390/block/dcssblk.c @@ -685,7 +685,7 @@ dcssblk_add_store(struct device *dev, struct device_attribute *attr, const char } get_device(&dev_info->dev); - device_add_disk(&dev_info->dev, dev_info->gd); + device_add_disk(&dev_info->dev, dev_info->gd, NULL); switch (dev_info->segment_type) { case SEG_TYPE_SR: diff --git a/drivers/s390/block/scm_blk.c b/drivers/s390/block/scm_blk.c index 98f66b7b6794..e01889394c84 100644 --- a/drivers/s390/block/scm_blk.c +++ b/drivers/s390/block/scm_blk.c @@ -500,7 +500,7 @@ int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev) /* 512 byte sectors */ set_capacity(bdev->gendisk, scmdev->size >> 9); - device_add_disk(&scmdev->dev, bdev->gendisk); + device_add_disk(&scmdev->dev, bdev->gendisk, NULL); return 0; out_queue: diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index eb97d2dd3651..62348412ed1b 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -3046,11 +3046,14 @@ scsi_device_quiesce(struct scsi_device *sdev) */ WARN_ON_ONCE(sdev->quiesced_by && sdev->quiesced_by != current); - blk_set_preempt_only(q); + if (sdev->quiesced_by == current) + return 0; + + blk_set_pm_only(q); blk_mq_freeze_queue(q); /* - * Ensure that the effect of blk_set_preempt_only() will be visible + * Ensure that the effect of blk_set_pm_only() will be visible * for percpu_ref_tryget() callers that occur after the queue * unfreeze even if the queue was already frozen before this function * was called. See also https://lwn.net/Articles/573497/. @@ -3063,7 +3066,7 @@ scsi_device_quiesce(struct scsi_device *sdev) if (err == 0) sdev->quiesced_by = current; else - blk_clear_preempt_only(q); + blk_clear_pm_only(q); mutex_unlock(&sdev->state_mutex); return err; @@ -3088,7 +3091,7 @@ void scsi_device_resume(struct scsi_device *sdev) mutex_lock(&sdev->state_mutex); WARN_ON_ONCE(!sdev->quiesced_by); sdev->quiesced_by = NULL; - blk_clear_preempt_only(sdev->request_queue); + blk_clear_pm_only(sdev->request_queue); if (sdev->sdev_state == SDEV_QUIESCE) scsi_device_set_state(sdev, SDEV_RUNNING); mutex_unlock(&sdev->state_mutex); diff --git a/drivers/scsi/scsi_pm.c b/drivers/scsi/scsi_pm.c index b44c1bb687a2..a2b4179bfdf7 100644 --- a/drivers/scsi/scsi_pm.c +++ b/drivers/scsi/scsi_pm.c @@ -8,6 +8,7 @@ #include <linux/pm_runtime.h> #include <linux/export.h> #include <linux/async.h> +#include <linux/blk-pm.h> #include <scsi/scsi.h> #include <scsi/scsi_device.h> diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index 4a57ffecc7e6..b762d0fd773c 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -45,6 +45,7 @@ #include <linux/init.h> #include <linux/blkdev.h> #include <linux/blkpg.h> +#include <linux/blk-pm.h> #include <linux/delay.h> #include <linux/mutex.h> #include <linux/string_helpers.h> @@ -3275,7 +3276,7 @@ static void sd_probe_async(void *data, async_cookie_t cookie) } blk_pm_runtime_init(sdp->request_queue, dev); - device_add_disk(dev, gd); + device_add_disk(dev, gd, NULL); if (sdkp->capacity) sd_dif_config_host(sdkp); diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c index d0389b20574d..54dd70ae9731 100644 --- a/drivers/scsi/sr.c +++ b/drivers/scsi/sr.c @@ -43,6 +43,7 @@ #include <linux/interrupt.h> #include <linux/init.h> #include <linux/blkdev.h> +#include <linux/blk-pm.h> #include <linux/mutex.h> #include <linux/slab.h> #include <linux/pm_runtime.h> @@ -758,7 +759,7 @@ static int sr_probe(struct device *dev) dev_set_drvdata(dev, cd); disk->flags |= GENHD_FL_REMOVABLE; - device_add_disk(&sdev->sdev_gendev, disk); + device_add_disk(&sdev->sdev_gendev, disk, NULL); sdev_printk(KERN_DEBUG, sdev, "Attached scsi CD-ROM %s\n", cd->cdi.name); diff --git a/drivers/soc/qcom/qcom-geni-se.c b/drivers/soc/qcom/qcom-geni-se.c index feed3db21c10..ee89ffb6dde8 100644 --- a/drivers/soc/qcom/qcom-geni-se.c +++ b/drivers/soc/qcom/qcom-geni-se.c @@ -513,7 +513,7 @@ EXPORT_SYMBOL(geni_se_resources_on); */ int geni_se_clk_tbl_get(struct geni_se *se, unsigned long **tbl) { - unsigned long freq = 0; + long freq = 0; int i; if (se->clk_perf_tbl) { @@ -529,7 +529,7 @@ int geni_se_clk_tbl_get(struct geni_se *se, unsigned long **tbl) for (i = 0; i < MAX_CLK_PERF_LEVEL; i++) { freq = clk_round_rate(se->clk, freq + 1); - if (!freq || freq == se->clk_perf_tbl[i - 1]) + if (freq <= 0 || freq == se->clk_perf_tbl[i - 1]) break; se->clk_perf_tbl[i] = freq; } @@ -544,16 +544,17 @@ EXPORT_SYMBOL(geni_se_clk_tbl_get); * @se: Pointer to the concerned serial engine. * @req_freq: Requested clock frequency. * @index: Index of the resultant frequency in the table. - * @res_freq: Resultant frequency which matches or is closer to the - * requested frequency. + * @res_freq: Resultant frequency of the source clock. * @exact: Flag to indicate exact multiple requirement of the requested * frequency. * - * This function is called by the protocol drivers to determine the matching - * or exact multiple of the requested frequency, as provided by the serial - * engine clock in order to meet the performance requirements. If there is - * no matching or exact multiple of the requested frequency found, then it - * selects the closest floor frequency, if exact flag is not set. + * This function is called by the protocol drivers to determine the best match + * of the requested frequency as provided by the serial engine clock in order + * to meet the performance requirements. + * + * If we return success: + * - if @exact is true then @res_freq / <an_integer> == @req_freq + * - if @exact is false then @res_freq / <an_integer> <= @req_freq * * Return: 0 on success, standard Linux error codes on failure. */ @@ -564,6 +565,9 @@ int geni_se_clk_freq_match(struct geni_se *se, unsigned long req_freq, unsigned long *tbl; int num_clk_levels; int i; + unsigned long best_delta; + unsigned long new_delta; + unsigned int divider; num_clk_levels = geni_se_clk_tbl_get(se, &tbl); if (num_clk_levels < 0) @@ -572,18 +576,21 @@ int geni_se_clk_freq_match(struct geni_se *se, unsigned long req_freq, if (num_clk_levels == 0) return -EINVAL; - *res_freq = 0; + best_delta = ULONG_MAX; for (i = 0; i < num_clk_levels; i++) { - if (!(tbl[i] % req_freq)) { + divider = DIV_ROUND_UP(tbl[i], req_freq); + new_delta = req_freq - tbl[i] / divider; + if (new_delta < best_delta) { + /* We have a new best! */ *index = i; *res_freq = tbl[i]; - return 0; - } - if (!(*res_freq) || ((tbl[i] > *res_freq) && - (tbl[i] < req_freq))) { - *index = i; - *res_freq = tbl[i]; + /* If the new best is exact then we're done */ + if (new_delta == 0) + return 0; + + /* Record how close we got */ + best_delta = new_delta; } } diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index 671d078349cc..f756450a8914 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -129,7 +129,7 @@ config SPI_BCM63XX config SPI_BCM63XX_HSSPI tristate "Broadcom BCM63XX HS SPI controller driver" - depends on BCM63XX || COMPILE_TEST + depends on BCM63XX || ARCH_BCM_63XX || COMPILE_TEST help This enables support for the High Speed SPI controller present on newer Broadcom BCM63XX SoCs. @@ -520,6 +520,12 @@ config SPI_RSPI help SPI driver for Renesas RSPI and QSPI blocks. +config SPI_QCOM_QSPI + tristate "QTI QSPI controller" + depends on ARCH_QCOM + help + QSPI(Quad SPI) driver for Qualcomm QSPI controller. + config SPI_QUP tristate "Qualcomm SPI controller with QUP interface" depends on ARCH_QCOM || (ARM && COMPILE_TEST) @@ -533,6 +539,18 @@ config SPI_QUP This driver can also be built as a module. If so, the module will be called spi_qup. +config SPI_QCOM_GENI + tristate "Qualcomm GENI based SPI controller" + depends on QCOM_GENI_SE + help + This driver supports GENI serial engine based SPI controller in + master mode on the Qualcomm Technologies Inc.'s SoCs. If you say + yes to this option, support will be included for the built-in SPI + interface on the Qualcomm Technologies Inc.'s SoCs. + + This driver can also be built as a module. If so, the module + will be called spi-geni-qcom. + config SPI_S3C24XX tristate "Samsung S3C24XX series SPI" depends on ARCH_S3C24XX @@ -596,6 +614,22 @@ config SPI_SIRF help SPI driver for CSR SiRFprimaII SoCs +config SPI_SLAVE_MT27XX + tristate "MediaTek SPI slave device" + depends on ARCH_MEDIATEK || COMPILE_TEST + depends on SPI_SLAVE + help + This selects the MediaTek(R) SPI slave device driver. + If you want to use MediaTek(R) SPI slave interface, + say Y or M here.If you are not sure, say N. + SPI slave drivers for Mediatek MT27XX series ARM SoCs. + +config SPI_SPRD + tristate "Spreadtrum SPI controller" + depends on ARCH_SPRD || COMPILE_TEST + help + SPI driver for Spreadtrum SoCs. + config SPI_SPRD_ADI tristate "Spreadtrum ADI controller" depends on ARCH_SPRD || COMPILE_TEST @@ -613,6 +647,15 @@ config SPI_STM32 is not available, the driver automatically falls back to PIO mode. +config SPI_STM32_QSPI + tristate "STMicroelectronics STM32 QUAD SPI controller" + depends on ARCH_STM32 || COMPILE_TEST + depends on OF + help + This enables support for the Quad SPI controller in master mode. + This driver does not support generic SPI. The implementation only + supports spi-mem interface. + config SPI_ST_SSC4 tristate "STMicroelectronics SPI SSC-based driver" depends on ARCH_STI || COMPILE_TEST diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile index a90d55970036..df04dfbe7d70 100644 --- a/drivers/spi/Makefile +++ b/drivers/spi/Makefile @@ -74,6 +74,8 @@ obj-$(CONFIG_SPI_PPC4xx) += spi-ppc4xx.o spi-pxa2xx-platform-objs := spi-pxa2xx.o spi-pxa2xx-dma.o obj-$(CONFIG_SPI_PXA2XX) += spi-pxa2xx-platform.o obj-$(CONFIG_SPI_PXA2XX_PCI) += spi-pxa2xx-pci.o +obj-$(CONFIG_SPI_QCOM_GENI) += spi-geni-qcom.o +obj-$(CONFIG_SPI_QCOM_QSPI) += spi-qcom-qspi.o obj-$(CONFIG_SPI_QUP) += spi-qup.o obj-$(CONFIG_SPI_ROCKCHIP) += spi-rockchip.o obj-$(CONFIG_SPI_RB4XX) += spi-rb4xx.o @@ -88,8 +90,11 @@ obj-$(CONFIG_SPI_SH_HSPI) += spi-sh-hspi.o obj-$(CONFIG_SPI_SH_MSIOF) += spi-sh-msiof.o obj-$(CONFIG_SPI_SH_SCI) += spi-sh-sci.o obj-$(CONFIG_SPI_SIRF) += spi-sirf.o +obj-$(CONFIG_SPI_SLAVE_MT27XX) += spi-slave-mt27xx.o +obj-$(CONFIG_SPI_SPRD) += spi-sprd.o obj-$(CONFIG_SPI_SPRD_ADI) += spi-sprd-adi.o obj-$(CONFIG_SPI_STM32) += spi-stm32.o +obj-$(CONFIG_SPI_STM32_QSPI) += spi-stm32-qspi.o obj-$(CONFIG_SPI_ST_SSC4) += spi-st-ssc4.o obj-$(CONFIG_SPI_SUN4I) += spi-sun4i.o obj-$(CONFIG_SPI_SUN6I) += spi-sun6i.o diff --git a/drivers/spi/spi-atmel.c b/drivers/spi/spi-atmel.c index 3f890d162934..74fddcd3282b 100644 --- a/drivers/spi/spi-atmel.c +++ b/drivers/spi/spi-atmel.c @@ -1767,10 +1767,8 @@ static int atmel_spi_suspend(struct device *dev) /* Stop the queue running */ ret = spi_master_suspend(master); - if (ret) { - dev_warn(dev, "cannot suspend master\n"); + if (ret) return ret; - } if (!pm_runtime_suspended(dev)) atmel_spi_runtime_suspend(dev); @@ -1799,11 +1797,7 @@ static int atmel_spi_resume(struct device *dev) } /* Start the queue running */ - ret = spi_master_resume(master); - if (ret) - dev_err(dev, "problem starting queue (%d)\n", ret); - - return ret; + return spi_master_resume(master); } #endif diff --git a/drivers/spi/spi-bcm-qspi.c b/drivers/spi/spi-bcm-qspi.c index 8612525fa4e3..584bcb018a62 100644 --- a/drivers/spi/spi-bcm-qspi.c +++ b/drivers/spi/spi-bcm-qspi.c @@ -89,7 +89,7 @@ #define BSPI_BPP_MODE_SELECT_MASK BIT(8) #define BSPI_BPP_ADDR_SELECT_MASK BIT(16) -#define BSPI_READ_LENGTH 512 +#define BSPI_READ_LENGTH 256 /* MSPI register offsets */ #define MSPI_SPCR0_LSB 0x000 @@ -355,7 +355,7 @@ static int bcm_qspi_bspi_set_flex_mode(struct bcm_qspi *qspi, int bpc = 0, bpp = 0; u8 command = op->cmd.opcode; int width = op->cmd.buswidth ? op->cmd.buswidth : SPI_NBITS_SINGLE; - int addrlen = op->addr.nbytes * 8; + int addrlen = op->addr.nbytes; int flex_mode = 1; dev_dbg(&qspi->pdev->dev, "set flex mode w %x addrlen %x hp %d\n", diff --git a/drivers/spi/spi-bcm63xx-hsspi.c b/drivers/spi/spi-bcm63xx-hsspi.c index c23849f7aa7b..9a06ffdb73b8 100644 --- a/drivers/spi/spi-bcm63xx-hsspi.c +++ b/drivers/spi/spi-bcm63xx-hsspi.c @@ -101,6 +101,7 @@ struct bcm63xx_hsspi { struct platform_device *pdev; struct clk *clk; + struct clk *pll_clk; void __iomem *regs; u8 __iomem *fifo; @@ -332,7 +333,7 @@ static int bcm63xx_hsspi_probe(struct platform_device *pdev) struct resource *res_mem; void __iomem *regs; struct device *dev = &pdev->dev; - struct clk *clk; + struct clk *clk, *pll_clk = NULL; int irq, ret; u32 reg, rate, num_cs = HSSPI_SPI_MAX_CS; @@ -358,7 +359,7 @@ static int bcm63xx_hsspi_probe(struct platform_device *pdev) rate = clk_get_rate(clk); if (!rate) { - struct clk *pll_clk = devm_clk_get(dev, "pll"); + pll_clk = devm_clk_get(dev, "pll"); if (IS_ERR(pll_clk)) { ret = PTR_ERR(pll_clk); @@ -373,19 +374,20 @@ static int bcm63xx_hsspi_probe(struct platform_device *pdev) clk_disable_unprepare(pll_clk); if (!rate) { ret = -EINVAL; - goto out_disable_clk; + goto out_disable_pll_clk; } } master = spi_alloc_master(&pdev->dev, sizeof(*bs)); if (!master) { ret = -ENOMEM; - goto out_disable_clk; + goto out_disable_pll_clk; } bs = spi_master_get_devdata(master); bs->pdev = pdev; bs->clk = clk; + bs->pll_clk = pll_clk; bs->regs = regs; bs->speed_hz = rate; bs->fifo = (u8 __iomem *)(bs->regs + HSSPI_FIFO_REG(0)); @@ -440,6 +442,8 @@ static int bcm63xx_hsspi_probe(struct platform_device *pdev) out_put_master: spi_master_put(master); +out_disable_pll_clk: + clk_disable_unprepare(pll_clk); out_disable_clk: clk_disable_unprepare(clk); return ret; @@ -453,6 +457,7 @@ static int bcm63xx_hsspi_remove(struct platform_device *pdev) /* reset the hardware and block queue progress */ __raw_writel(0, bs->regs + HSSPI_INT_MASK_REG); + clk_disable_unprepare(bs->pll_clk); clk_disable_unprepare(bs->clk); return 0; @@ -465,6 +470,7 @@ static int bcm63xx_hsspi_suspend(struct device *dev) struct bcm63xx_hsspi *bs = spi_master_get_devdata(master); spi_master_suspend(master); + clk_disable_unprepare(bs->pll_clk); clk_disable_unprepare(bs->clk); return 0; @@ -480,6 +486,12 @@ static int bcm63xx_hsspi_resume(struct device *dev) if (ret) return ret; + if (bs->pll_clk) { + ret = clk_prepare_enable(bs->pll_clk); + if (ret) + return ret; + } + spi_master_resume(master); return 0; diff --git a/drivers/spi/spi-davinci.c b/drivers/spi/spi-davinci.c index a02099c90c5c..56adec83f8fc 100644 --- a/drivers/spi/spi-davinci.c +++ b/drivers/spi/spi-davinci.c @@ -208,13 +208,11 @@ static inline void clear_io_bits(void __iomem *addr, u32 bits) static void davinci_spi_chipselect(struct spi_device *spi, int value) { struct davinci_spi *dspi; - struct davinci_spi_platform_data *pdata; struct davinci_spi_config *spicfg = spi->controller_data; u8 chip_sel = spi->chip_select; u16 spidat1 = CS_DEFAULT; dspi = spi_master_get_devdata(spi->master); - pdata = &dspi->pdata; /* program delay transfers if tx_delay is non zero */ if (spicfg && spicfg->wdelay) @@ -232,7 +230,8 @@ static void davinci_spi_chipselect(struct spi_device *spi, int value) !(spi->mode & SPI_CS_HIGH)); } else { if (value == BITBANG_CS_ACTIVE) { - spidat1 |= SPIDAT1_CSHOLD_MASK; + if (!(spi->mode & SPI_CS_WORD)) + spidat1 |= SPIDAT1_CSHOLD_MASK; spidat1 &= ~(0x1 << chip_sel); } } @@ -421,26 +420,17 @@ static int davinci_spi_setup(struct spi_device *spi) { int retval = 0; struct davinci_spi *dspi; - struct davinci_spi_platform_data *pdata; struct spi_master *master = spi->master; struct device_node *np = spi->dev.of_node; bool internal_cs = true; dspi = spi_master_get_devdata(spi->master); - pdata = &dspi->pdata; if (!(spi->mode & SPI_NO_CS)) { if (np && (master->cs_gpios != NULL) && (spi->cs_gpio >= 0)) { retval = gpio_direction_output( spi->cs_gpio, !(spi->mode & SPI_CS_HIGH)); internal_cs = false; - } else if (pdata->chip_sel && - spi->chip_select < pdata->num_chipselect && - pdata->chip_sel[spi->chip_select] != SPI_INTERN_CS) { - spi->cs_gpio = pdata->chip_sel[spi->chip_select]; - retval = gpio_direction_output( - spi->cs_gpio, !(spi->mode & SPI_CS_HIGH)); - internal_cs = false; } if (retval) { @@ -449,8 +439,9 @@ static int davinci_spi_setup(struct spi_device *spi) return retval; } - if (internal_cs) + if (internal_cs) { set_io_bits(dspi->base + SPIPC0, 1 << spi->chip_select); + } } if (spi->mode & SPI_READY) @@ -985,7 +976,7 @@ static int davinci_spi_probe(struct platform_device *pdev) dspi->prescaler_limit = pdata->prescaler_limit; dspi->version = pdata->version; - dspi->bitbang.flags = SPI_NO_CS | SPI_LSB_FIRST | SPI_LOOP; + dspi->bitbang.flags = SPI_NO_CS | SPI_LSB_FIRST | SPI_LOOP | SPI_CS_WORD; if (dspi->version == SPI_VERSION_2) dspi->bitbang.flags |= SPI_READY; diff --git a/drivers/spi/spi-dw-mmio.c b/drivers/spi/spi-dw-mmio.c index e80f60ed6fdf..3ffb6a40fe0c 100644 --- a/drivers/spi/spi-dw-mmio.c +++ b/drivers/spi/spi-dw-mmio.c @@ -34,8 +34,9 @@ struct dw_spi_mmio { }; #define MSCC_CPU_SYSTEM_CTRL_GENERAL_CTRL 0x24 -#define OCELOT_IF_SI_OWNER_MASK GENMASK(5, 4) #define OCELOT_IF_SI_OWNER_OFFSET 4 +#define JAGUAR2_IF_SI_OWNER_OFFSET 6 +#define MSCC_IF_SI_OWNER_MASK GENMASK(1, 0) #define MSCC_IF_SI_OWNER_SISL 0 #define MSCC_IF_SI_OWNER_SIBM 1 #define MSCC_IF_SI_OWNER_SIMC 2 @@ -76,7 +77,8 @@ static void dw_spi_mscc_set_cs(struct spi_device *spi, bool enable) } static int dw_spi_mscc_init(struct platform_device *pdev, - struct dw_spi_mmio *dwsmmio) + struct dw_spi_mmio *dwsmmio, + const char *cpu_syscon, u32 if_si_owner_offset) { struct dw_spi_mscc *dwsmscc; struct resource *res; @@ -92,7 +94,7 @@ static int dw_spi_mscc_init(struct platform_device *pdev, return PTR_ERR(dwsmscc->spi_mst); } - dwsmscc->syscon = syscon_regmap_lookup_by_compatible("mscc,ocelot-cpu-syscon"); + dwsmscc->syscon = syscon_regmap_lookup_by_compatible(cpu_syscon); if (IS_ERR(dwsmscc->syscon)) return PTR_ERR(dwsmscc->syscon); @@ -101,8 +103,8 @@ static int dw_spi_mscc_init(struct platform_device *pdev, /* Select the owner of the SI interface */ regmap_update_bits(dwsmscc->syscon, MSCC_CPU_SYSTEM_CTRL_GENERAL_CTRL, - OCELOT_IF_SI_OWNER_MASK, - MSCC_IF_SI_OWNER_SIMC << OCELOT_IF_SI_OWNER_OFFSET); + MSCC_IF_SI_OWNER_MASK << if_si_owner_offset, + MSCC_IF_SI_OWNER_SIMC << if_si_owner_offset); dwsmmio->dws.set_cs = dw_spi_mscc_set_cs; dwsmmio->priv = dwsmscc; @@ -110,6 +112,28 @@ static int dw_spi_mscc_init(struct platform_device *pdev, return 0; } +static int dw_spi_mscc_ocelot_init(struct platform_device *pdev, + struct dw_spi_mmio *dwsmmio) +{ + return dw_spi_mscc_init(pdev, dwsmmio, "mscc,ocelot-cpu-syscon", + OCELOT_IF_SI_OWNER_OFFSET); +} + +static int dw_spi_mscc_jaguar2_init(struct platform_device *pdev, + struct dw_spi_mmio *dwsmmio) +{ + return dw_spi_mscc_init(pdev, dwsmmio, "mscc,jaguar2-cpu-syscon", + JAGUAR2_IF_SI_OWNER_OFFSET); +} + +static int dw_spi_alpine_init(struct platform_device *pdev, + struct dw_spi_mmio *dwsmmio) +{ + dwsmmio->dws.cs_override = 1; + + return 0; +} + static int dw_spi_mmio_probe(struct platform_device *pdev) { int (*init_func)(struct platform_device *pdev, @@ -212,7 +236,9 @@ static int dw_spi_mmio_remove(struct platform_device *pdev) static const struct of_device_id dw_spi_mmio_of_match[] = { { .compatible = "snps,dw-apb-ssi", }, - { .compatible = "mscc,ocelot-spi", .data = dw_spi_mscc_init}, + { .compatible = "mscc,ocelot-spi", .data = dw_spi_mscc_ocelot_init}, + { .compatible = "mscc,jaguar2-spi", .data = dw_spi_mscc_jaguar2_init}, + { .compatible = "amazon,alpine-dw-apb-ssi", .data = dw_spi_alpine_init}, { /* end of table */} }; MODULE_DEVICE_TABLE(of, dw_spi_mmio_of_match); diff --git a/drivers/spi/spi-dw.c b/drivers/spi/spi-dw.c index ac2eb89ef7a5..b705f2bdb8b9 100644 --- a/drivers/spi/spi-dw.c +++ b/drivers/spi/spi-dw.c @@ -144,6 +144,8 @@ void dw_spi_set_cs(struct spi_device *spi, bool enable) if (!enable) dw_writel(dws, DW_SPI_SER, BIT(spi->chip_select)); + else if (dws->cs_override) + dw_writel(dws, DW_SPI_SER, 0); } EXPORT_SYMBOL_GPL(dw_spi_set_cs); @@ -308,15 +310,10 @@ static int dw_spi_transfer_one(struct spi_controller *master, dws->current_freq = transfer->speed_hz; spi_set_clk(dws, chip->clk_div); } - if (transfer->bits_per_word == 8) { - dws->n_bytes = 1; - dws->dma_width = 1; - } else if (transfer->bits_per_word == 16) { - dws->n_bytes = 2; - dws->dma_width = 2; - } else { - return -EINVAL; - } + + dws->n_bytes = DIV_ROUND_UP(transfer->bits_per_word, BITS_PER_BYTE); + dws->dma_width = DIV_ROUND_UP(transfer->bits_per_word, BITS_PER_BYTE); + /* Default SPI mode is SCPOL = 0, SCPH = 0 */ cr0 = (transfer->bits_per_word - 1) | (chip->type << SPI_FRF_OFFSET) @@ -468,6 +465,10 @@ static void spi_hw_init(struct device *dev, struct dw_spi *dws) dws->fifo_len = (fifo == 1) ? 0 : fifo; dev_dbg(dev, "Detected FIFO size: %u bytes\n", dws->fifo_len); } + + /* enable HW fixup for explicit CS deselect for Amazon's alpine chip */ + if (dws->cs_override) + dw_writel(dws, DW_SPI_CS_OVERRIDE, 0xF); } int dw_spi_add_host(struct device *dev, struct dw_spi *dws) @@ -496,7 +497,7 @@ int dw_spi_add_host(struct device *dev, struct dw_spi *dws) } master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LOOP; - master->bits_per_word_mask = SPI_BPW_MASK(8) | SPI_BPW_MASK(16); + master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 16); master->bus_num = dws->bus_num; master->num_chipselect = dws->num_cs; master->setup = dw_spi_setup; @@ -572,13 +573,8 @@ EXPORT_SYMBOL_GPL(dw_spi_suspend_host); int dw_spi_resume_host(struct dw_spi *dws) { - int ret; - spi_hw_init(&dws->master->dev, dws); - ret = spi_controller_resume(dws->master); - if (ret) - dev_err(&dws->master->dev, "fail to start queue (%d)\n", ret); - return ret; + return spi_controller_resume(dws->master); } EXPORT_SYMBOL_GPL(dw_spi_resume_host); diff --git a/drivers/spi/spi-dw.h b/drivers/spi/spi-dw.h index 0168b08364d5..c9c15881e982 100644 --- a/drivers/spi/spi-dw.h +++ b/drivers/spi/spi-dw.h @@ -32,6 +32,7 @@ #define DW_SPI_IDR 0x58 #define DW_SPI_VERSION 0x5c #define DW_SPI_DR 0x60 +#define DW_SPI_CS_OVERRIDE 0xf4 /* Bit fields in CTRLR0 */ #define SPI_DFS_OFFSET 0 @@ -109,6 +110,7 @@ struct dw_spi { u32 fifo_len; /* depth of the FIFO buffer */ u32 max_freq; /* max bus freq supported */ + int cs_override; u32 reg_io_width; /* DR I/O width in bytes */ u16 bus_num; u16 num_cs; /* supported slave numbers */ diff --git a/drivers/spi/spi-ep93xx.c b/drivers/spi/spi-ep93xx.c index f1526757aaf6..79fc3940245a 100644 --- a/drivers/spi/spi-ep93xx.c +++ b/drivers/spi/spi-ep93xx.c @@ -246,6 +246,19 @@ static int ep93xx_spi_read_write(struct spi_master *master) return -EINPROGRESS; } +static enum dma_transfer_direction +ep93xx_dma_data_to_trans_dir(enum dma_data_direction dir) +{ + switch (dir) { + case DMA_TO_DEVICE: + return DMA_MEM_TO_DEV; + case DMA_FROM_DEVICE: + return DMA_DEV_TO_MEM; + default: + return DMA_TRANS_NONE; + } +} + /** * ep93xx_spi_dma_prepare() - prepares a DMA transfer * @master: SPI master @@ -257,7 +270,7 @@ static int ep93xx_spi_read_write(struct spi_master *master) */ static struct dma_async_tx_descriptor * ep93xx_spi_dma_prepare(struct spi_master *master, - enum dma_transfer_direction dir) + enum dma_data_direction dir) { struct ep93xx_spi *espi = spi_master_get_devdata(master); struct spi_transfer *xfer = master->cur_msg->state; @@ -277,9 +290,9 @@ ep93xx_spi_dma_prepare(struct spi_master *master, buswidth = DMA_SLAVE_BUSWIDTH_1_BYTE; memset(&conf, 0, sizeof(conf)); - conf.direction = dir; + conf.direction = ep93xx_dma_data_to_trans_dir(dir); - if (dir == DMA_DEV_TO_MEM) { + if (dir == DMA_FROM_DEVICE) { chan = espi->dma_rx; buf = xfer->rx_buf; sgt = &espi->rx_sgt; @@ -343,7 +356,8 @@ ep93xx_spi_dma_prepare(struct spi_master *master, if (!nents) return ERR_PTR(-ENOMEM); - txd = dmaengine_prep_slave_sg(chan, sgt->sgl, nents, dir, DMA_CTRL_ACK); + txd = dmaengine_prep_slave_sg(chan, sgt->sgl, nents, conf.direction, + DMA_CTRL_ACK); if (!txd) { dma_unmap_sg(chan->device->dev, sgt->sgl, sgt->nents, dir); return ERR_PTR(-ENOMEM); @@ -360,13 +374,13 @@ ep93xx_spi_dma_prepare(struct spi_master *master, * unmapped. */ static void ep93xx_spi_dma_finish(struct spi_master *master, - enum dma_transfer_direction dir) + enum dma_data_direction dir) { struct ep93xx_spi *espi = spi_master_get_devdata(master); struct dma_chan *chan; struct sg_table *sgt; - if (dir == DMA_DEV_TO_MEM) { + if (dir == DMA_FROM_DEVICE) { chan = espi->dma_rx; sgt = &espi->rx_sgt; } else { @@ -381,8 +395,8 @@ static void ep93xx_spi_dma_callback(void *callback_param) { struct spi_master *master = callback_param; - ep93xx_spi_dma_finish(master, DMA_MEM_TO_DEV); - ep93xx_spi_dma_finish(master, DMA_DEV_TO_MEM); + ep93xx_spi_dma_finish(master, DMA_TO_DEVICE); + ep93xx_spi_dma_finish(master, DMA_FROM_DEVICE); spi_finalize_current_transfer(master); } @@ -392,15 +406,15 @@ static int ep93xx_spi_dma_transfer(struct spi_master *master) struct ep93xx_spi *espi = spi_master_get_devdata(master); struct dma_async_tx_descriptor *rxd, *txd; - rxd = ep93xx_spi_dma_prepare(master, DMA_DEV_TO_MEM); + rxd = ep93xx_spi_dma_prepare(master, DMA_FROM_DEVICE); if (IS_ERR(rxd)) { dev_err(&master->dev, "DMA RX failed: %ld\n", PTR_ERR(rxd)); return PTR_ERR(rxd); } - txd = ep93xx_spi_dma_prepare(master, DMA_MEM_TO_DEV); + txd = ep93xx_spi_dma_prepare(master, DMA_TO_DEVICE); if (IS_ERR(txd)) { - ep93xx_spi_dma_finish(master, DMA_DEV_TO_MEM); + ep93xx_spi_dma_finish(master, DMA_FROM_DEVICE); dev_err(&master->dev, "DMA TX failed: %ld\n", PTR_ERR(txd)); return PTR_ERR(txd); } diff --git a/drivers/spi/spi-fsl-espi.c b/drivers/spi/spi-fsl-espi.c index 1e8ff6256079..cf2118dc91f4 100644 --- a/drivers/spi/spi-fsl-espi.c +++ b/drivers/spi/spi-fsl-espi.c @@ -798,10 +798,8 @@ static int of_fsl_espi_suspend(struct device *dev) int ret; ret = spi_master_suspend(master); - if (ret) { - dev_warn(dev, "cannot suspend master\n"); + if (ret) return ret; - } return pm_runtime_force_suspend(dev); } diff --git a/drivers/spi/spi-fsl-lpspi.c b/drivers/spi/spi-fsl-lpspi.c index e6d5cc6ab108..51670976faa3 100644 --- a/drivers/spi/spi-fsl-lpspi.c +++ b/drivers/spi/spi-fsl-lpspi.c @@ -276,7 +276,7 @@ static int fsl_lpspi_config(struct fsl_lpspi_data *fsl_lpspi) fsl_lpspi_set_watermark(fsl_lpspi); - temp = CFGR1_PCSCFG | CFGR1_MASTER | CFGR1_NOSTALL; + temp = CFGR1_PCSCFG | CFGR1_MASTER; if (fsl_lpspi->config.mode & SPI_CS_HIGH) temp |= CFGR1_PCSPOL; writel(temp, fsl_lpspi->base + IMX7ULP_CFGR1); diff --git a/drivers/spi/spi-geni-qcom.c b/drivers/spi/spi-geni-qcom.c new file mode 100644 index 000000000000..6432ecc4e2ca --- /dev/null +++ b/drivers/spi/spi-geni-qcom.c @@ -0,0 +1,703 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2017-2018, The Linux foundation. All rights reserved. + +#include <linux/clk.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/log2.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/qcom-geni-se.h> +#include <linux/spi/spi.h> +#include <linux/spinlock.h> + +/* SPI SE specific registers and respective register fields */ +#define SE_SPI_CPHA 0x224 +#define CPHA BIT(0) + +#define SE_SPI_LOOPBACK 0x22c +#define LOOPBACK_ENABLE 0x1 +#define NORMAL_MODE 0x0 +#define LOOPBACK_MSK GENMASK(1, 0) + +#define SE_SPI_CPOL 0x230 +#define CPOL BIT(2) + +#define SE_SPI_DEMUX_OUTPUT_INV 0x24c +#define CS_DEMUX_OUTPUT_INV_MSK GENMASK(3, 0) + +#define SE_SPI_DEMUX_SEL 0x250 +#define CS_DEMUX_OUTPUT_SEL GENMASK(3, 0) + +#define SE_SPI_TRANS_CFG 0x25c +#define CS_TOGGLE BIT(0) + +#define SE_SPI_WORD_LEN 0x268 +#define WORD_LEN_MSK GENMASK(9, 0) +#define MIN_WORD_LEN 4 + +#define SE_SPI_TX_TRANS_LEN 0x26c +#define SE_SPI_RX_TRANS_LEN 0x270 +#define TRANS_LEN_MSK GENMASK(23, 0) + +#define SE_SPI_PRE_POST_CMD_DLY 0x274 + +#define SE_SPI_DELAY_COUNTERS 0x278 +#define SPI_INTER_WORDS_DELAY_MSK GENMASK(9, 0) +#define SPI_CS_CLK_DELAY_MSK GENMASK(19, 10) +#define SPI_CS_CLK_DELAY_SHFT 10 + +/* M_CMD OP codes for SPI */ +#define SPI_TX_ONLY 1 +#define SPI_RX_ONLY 2 +#define SPI_FULL_DUPLEX 3 +#define SPI_TX_RX 7 +#define SPI_CS_ASSERT 8 +#define SPI_CS_DEASSERT 9 +#define SPI_SCK_ONLY 10 +/* M_CMD params for SPI */ +#define SPI_PRE_CMD_DELAY BIT(0) +#define TIMESTAMP_BEFORE BIT(1) +#define FRAGMENTATION BIT(2) +#define TIMESTAMP_AFTER BIT(3) +#define POST_CMD_DELAY BIT(4) + +/* SPI M_COMMAND OPCODE */ +enum spi_mcmd_code { + CMD_NONE, + CMD_XFER, + CMD_CS, + CMD_CANCEL, +}; + + +struct spi_geni_master { + struct geni_se se; + struct device *dev; + u32 tx_fifo_depth; + u32 fifo_width_bits; + u32 tx_wm; + unsigned long cur_speed_hz; + unsigned int cur_bits_per_word; + unsigned int tx_rem_bytes; + unsigned int rx_rem_bytes; + const struct spi_transfer *cur_xfer; + struct completion xfer_done; + unsigned int oversampling; + spinlock_t lock; + unsigned int cur_mcmd; + int irq; +}; + +static void handle_fifo_timeout(struct spi_master *spi, + struct spi_message *msg); + +static int get_spi_clk_cfg(unsigned int speed_hz, + struct spi_geni_master *mas, + unsigned int *clk_idx, + unsigned int *clk_div) +{ + unsigned long sclk_freq; + unsigned int actual_hz; + struct geni_se *se = &mas->se; + int ret; + + ret = geni_se_clk_freq_match(&mas->se, + speed_hz * mas->oversampling, + clk_idx, &sclk_freq, false); + if (ret) { + dev_err(mas->dev, "Failed(%d) to find src clk for %dHz\n", + ret, speed_hz); + return ret; + } + + *clk_div = DIV_ROUND_UP(sclk_freq, mas->oversampling * speed_hz); + actual_hz = sclk_freq / (mas->oversampling * *clk_div); + + dev_dbg(mas->dev, "req %u=>%u sclk %lu, idx %d, div %d\n", speed_hz, + actual_hz, sclk_freq, *clk_idx, *clk_div); + ret = clk_set_rate(se->clk, sclk_freq); + if (ret) + dev_err(mas->dev, "clk_set_rate failed %d\n", ret); + return ret; +} + +static void spi_geni_set_cs(struct spi_device *slv, bool set_flag) +{ + struct spi_geni_master *mas = spi_master_get_devdata(slv->master); + struct spi_master *spi = dev_get_drvdata(mas->dev); + struct geni_se *se = &mas->se; + unsigned long timeout; + + reinit_completion(&mas->xfer_done); + pm_runtime_get_sync(mas->dev); + if (!(slv->mode & SPI_CS_HIGH)) + set_flag = !set_flag; + + mas->cur_mcmd = CMD_CS; + if (set_flag) + geni_se_setup_m_cmd(se, SPI_CS_ASSERT, 0); + else + geni_se_setup_m_cmd(se, SPI_CS_DEASSERT, 0); + + timeout = wait_for_completion_timeout(&mas->xfer_done, HZ); + if (!timeout) + handle_fifo_timeout(spi, NULL); + + pm_runtime_put(mas->dev); +} + +static void spi_setup_word_len(struct spi_geni_master *mas, u16 mode, + unsigned int bits_per_word) +{ + unsigned int pack_words; + bool msb_first = (mode & SPI_LSB_FIRST) ? false : true; + struct geni_se *se = &mas->se; + u32 word_len; + + word_len = readl(se->base + SE_SPI_WORD_LEN); + + /* + * If bits_per_word isn't a byte aligned value, set the packing to be + * 1 SPI word per FIFO word. + */ + if (!(mas->fifo_width_bits % bits_per_word)) + pack_words = mas->fifo_width_bits / bits_per_word; + else + pack_words = 1; + word_len &= ~WORD_LEN_MSK; + word_len |= ((bits_per_word - MIN_WORD_LEN) & WORD_LEN_MSK); + geni_se_config_packing(&mas->se, bits_per_word, pack_words, msb_first, + true, true); + writel(word_len, se->base + SE_SPI_WORD_LEN); +} + +static int setup_fifo_params(struct spi_device *spi_slv, + struct spi_master *spi) +{ + struct spi_geni_master *mas = spi_master_get_devdata(spi); + struct geni_se *se = &mas->se; + u32 loopback_cfg, cpol, cpha, demux_output_inv; + u32 demux_sel, clk_sel, m_clk_cfg, idx, div; + int ret; + + loopback_cfg = readl(se->base + SE_SPI_LOOPBACK); + cpol = readl(se->base + SE_SPI_CPOL); + cpha = readl(se->base + SE_SPI_CPHA); + demux_output_inv = 0; + loopback_cfg &= ~LOOPBACK_MSK; + cpol &= ~CPOL; + cpha &= ~CPHA; + + if (spi_slv->mode & SPI_LOOP) + loopback_cfg |= LOOPBACK_ENABLE; + + if (spi_slv->mode & SPI_CPOL) + cpol |= CPOL; + + if (spi_slv->mode & SPI_CPHA) + cpha |= CPHA; + + if (spi_slv->mode & SPI_CS_HIGH) + demux_output_inv = BIT(spi_slv->chip_select); + + demux_sel = spi_slv->chip_select; + mas->cur_speed_hz = spi_slv->max_speed_hz; + mas->cur_bits_per_word = spi_slv->bits_per_word; + + ret = get_spi_clk_cfg(mas->cur_speed_hz, mas, &idx, &div); + if (ret) { + dev_err(mas->dev, "Err setting clks ret(%d) for %ld\n", + ret, mas->cur_speed_hz); + return ret; + } + + clk_sel = idx & CLK_SEL_MSK; + m_clk_cfg = (div << CLK_DIV_SHFT) | SER_CLK_EN; + spi_setup_word_len(mas, spi_slv->mode, spi_slv->bits_per_word); + writel(loopback_cfg, se->base + SE_SPI_LOOPBACK); + writel(demux_sel, se->base + SE_SPI_DEMUX_SEL); + writel(cpha, se->base + SE_SPI_CPHA); + writel(cpol, se->base + SE_SPI_CPOL); + writel(demux_output_inv, se->base + SE_SPI_DEMUX_OUTPUT_INV); + writel(clk_sel, se->base + SE_GENI_CLK_SEL); + writel(m_clk_cfg, se->base + GENI_SER_M_CLK_CFG); + return 0; +} + +static int spi_geni_prepare_message(struct spi_master *spi, + struct spi_message *spi_msg) +{ + int ret; + struct spi_geni_master *mas = spi_master_get_devdata(spi); + struct geni_se *se = &mas->se; + + geni_se_select_mode(se, GENI_SE_FIFO); + reinit_completion(&mas->xfer_done); + ret = setup_fifo_params(spi_msg->spi, spi); + if (ret) + dev_err(mas->dev, "Couldn't select mode %d\n", ret); + return ret; +} + +static int spi_geni_init(struct spi_geni_master *mas) +{ + struct geni_se *se = &mas->se; + unsigned int proto, major, minor, ver; + + pm_runtime_get_sync(mas->dev); + + proto = geni_se_read_proto(se); + if (proto != GENI_SE_SPI) { + dev_err(mas->dev, "Invalid proto %d\n", proto); + pm_runtime_put(mas->dev); + return -ENXIO; + } + mas->tx_fifo_depth = geni_se_get_tx_fifo_depth(se); + + /* Width of Tx and Rx FIFO is same */ + mas->fifo_width_bits = geni_se_get_tx_fifo_width(se); + + /* + * Hardware programming guide suggests to configure + * RX FIFO RFR level to fifo_depth-2. + */ + geni_se_init(se, 0x0, mas->tx_fifo_depth - 2); + /* Transmit an entire FIFO worth of data per IRQ */ + mas->tx_wm = 1; + ver = geni_se_get_qup_hw_version(se); + major = GENI_SE_VERSION_MAJOR(ver); + minor = GENI_SE_VERSION_MINOR(ver); + + if (major == 1 && minor == 0) + mas->oversampling = 2; + else + mas->oversampling = 1; + + pm_runtime_put(mas->dev); + return 0; +} + +static void setup_fifo_xfer(struct spi_transfer *xfer, + struct spi_geni_master *mas, + u16 mode, struct spi_master *spi) +{ + u32 m_cmd = 0; + u32 spi_tx_cfg, len; + struct geni_se *se = &mas->se; + + spi_tx_cfg = readl(se->base + SE_SPI_TRANS_CFG); + if (xfer->bits_per_word != mas->cur_bits_per_word) { + spi_setup_word_len(mas, mode, xfer->bits_per_word); + mas->cur_bits_per_word = xfer->bits_per_word; + } + + /* Speed and bits per word can be overridden per transfer */ + if (xfer->speed_hz != mas->cur_speed_hz) { + int ret; + u32 clk_sel, m_clk_cfg; + unsigned int idx, div; + + ret = get_spi_clk_cfg(xfer->speed_hz, mas, &idx, &div); + if (ret) { + dev_err(mas->dev, "Err setting clks:%d\n", ret); + return; + } + /* + * SPI core clock gets configured with the requested frequency + * or the frequency closer to the requested frequency. + * For that reason requested frequency is stored in the + * cur_speed_hz and referred in the consecutive transfer instead + * of calling clk_get_rate() API. + */ + mas->cur_speed_hz = xfer->speed_hz; + clk_sel = idx & CLK_SEL_MSK; + m_clk_cfg = (div << CLK_DIV_SHFT) | SER_CLK_EN; + writel(clk_sel, se->base + SE_GENI_CLK_SEL); + writel(m_clk_cfg, se->base + GENI_SER_M_CLK_CFG); + } + + mas->tx_rem_bytes = 0; + mas->rx_rem_bytes = 0; + if (xfer->tx_buf && xfer->rx_buf) + m_cmd = SPI_FULL_DUPLEX; + else if (xfer->tx_buf) + m_cmd = SPI_TX_ONLY; + else if (xfer->rx_buf) + m_cmd = SPI_RX_ONLY; + + spi_tx_cfg &= ~CS_TOGGLE; + + if (!(mas->cur_bits_per_word % MIN_WORD_LEN)) + len = xfer->len * BITS_PER_BYTE / mas->cur_bits_per_word; + else + len = xfer->len / (mas->cur_bits_per_word / BITS_PER_BYTE + 1); + len &= TRANS_LEN_MSK; + + mas->cur_xfer = xfer; + if (m_cmd & SPI_TX_ONLY) { + mas->tx_rem_bytes = xfer->len; + writel(len, se->base + SE_SPI_TX_TRANS_LEN); + } + + if (m_cmd & SPI_RX_ONLY) { + writel(len, se->base + SE_SPI_RX_TRANS_LEN); + mas->rx_rem_bytes = xfer->len; + } + writel(spi_tx_cfg, se->base + SE_SPI_TRANS_CFG); + mas->cur_mcmd = CMD_XFER; + geni_se_setup_m_cmd(se, m_cmd, FRAGMENTATION); + + /* + * TX_WATERMARK_REG should be set after SPI configuration and + * setting up GENI SE engine, as driver starts data transfer + * for the watermark interrupt. + */ + if (m_cmd & SPI_TX_ONLY) + writel(mas->tx_wm, se->base + SE_GENI_TX_WATERMARK_REG); +} + +static void handle_fifo_timeout(struct spi_master *spi, + struct spi_message *msg) +{ + struct spi_geni_master *mas = spi_master_get_devdata(spi); + unsigned long time_left, flags; + struct geni_se *se = &mas->se; + + spin_lock_irqsave(&mas->lock, flags); + reinit_completion(&mas->xfer_done); + mas->cur_mcmd = CMD_CANCEL; + geni_se_cancel_m_cmd(se); + writel(0, se->base + SE_GENI_TX_WATERMARK_REG); + spin_unlock_irqrestore(&mas->lock, flags); + time_left = wait_for_completion_timeout(&mas->xfer_done, HZ); + if (time_left) + return; + + spin_lock_irqsave(&mas->lock, flags); + reinit_completion(&mas->xfer_done); + geni_se_abort_m_cmd(se); + spin_unlock_irqrestore(&mas->lock, flags); + time_left = wait_for_completion_timeout(&mas->xfer_done, HZ); + if (!time_left) + dev_err(mas->dev, "Failed to cancel/abort m_cmd\n"); +} + +static int spi_geni_transfer_one(struct spi_master *spi, + struct spi_device *slv, + struct spi_transfer *xfer) +{ + struct spi_geni_master *mas = spi_master_get_devdata(spi); + + /* Terminate and return success for 0 byte length transfer */ + if (!xfer->len) + return 0; + + setup_fifo_xfer(xfer, mas, slv->mode, spi); + return 1; +} + +static unsigned int geni_byte_per_fifo_word(struct spi_geni_master *mas) +{ + /* + * Calculate how many bytes we'll put in each FIFO word. If the + * transfer words don't pack cleanly into a FIFO word we'll just put + * one transfer word in each FIFO word. If they do pack we'll pack 'em. + */ + if (mas->fifo_width_bits % mas->cur_bits_per_word) + return roundup_pow_of_two(DIV_ROUND_UP(mas->cur_bits_per_word, + BITS_PER_BYTE)); + + return mas->fifo_width_bits / BITS_PER_BYTE; +} + +static void geni_spi_handle_tx(struct spi_geni_master *mas) +{ + struct geni_se *se = &mas->se; + unsigned int max_bytes; + const u8 *tx_buf; + unsigned int bytes_per_fifo_word = geni_byte_per_fifo_word(mas); + unsigned int i = 0; + + max_bytes = (mas->tx_fifo_depth - mas->tx_wm) * bytes_per_fifo_word; + if (mas->tx_rem_bytes < max_bytes) + max_bytes = mas->tx_rem_bytes; + + tx_buf = mas->cur_xfer->tx_buf + mas->cur_xfer->len - mas->tx_rem_bytes; + while (i < max_bytes) { + unsigned int j; + unsigned int bytes_to_write; + u32 fifo_word = 0; + u8 *fifo_byte = (u8 *)&fifo_word; + + bytes_to_write = min(bytes_per_fifo_word, max_bytes - i); + for (j = 0; j < bytes_to_write; j++) + fifo_byte[j] = tx_buf[i++]; + iowrite32_rep(se->base + SE_GENI_TX_FIFOn, &fifo_word, 1); + } + mas->tx_rem_bytes -= max_bytes; + if (!mas->tx_rem_bytes) + writel(0, se->base + SE_GENI_TX_WATERMARK_REG); +} + +static void geni_spi_handle_rx(struct spi_geni_master *mas) +{ + struct geni_se *se = &mas->se; + u32 rx_fifo_status; + unsigned int rx_bytes; + unsigned int rx_last_byte_valid; + u8 *rx_buf; + unsigned int bytes_per_fifo_word = geni_byte_per_fifo_word(mas); + unsigned int i = 0; + + rx_fifo_status = readl(se->base + SE_GENI_RX_FIFO_STATUS); + rx_bytes = (rx_fifo_status & RX_FIFO_WC_MSK) * bytes_per_fifo_word; + if (rx_fifo_status & RX_LAST) { + rx_last_byte_valid = rx_fifo_status & RX_LAST_BYTE_VALID_MSK; + rx_last_byte_valid >>= RX_LAST_BYTE_VALID_SHFT; + if (rx_last_byte_valid && rx_last_byte_valid < 4) + rx_bytes -= bytes_per_fifo_word - rx_last_byte_valid; + } + if (mas->rx_rem_bytes < rx_bytes) + rx_bytes = mas->rx_rem_bytes; + + rx_buf = mas->cur_xfer->rx_buf + mas->cur_xfer->len - mas->rx_rem_bytes; + while (i < rx_bytes) { + u32 fifo_word = 0; + u8 *fifo_byte = (u8 *)&fifo_word; + unsigned int bytes_to_read; + unsigned int j; + + bytes_to_read = min(bytes_per_fifo_word, rx_bytes - i); + ioread32_rep(se->base + SE_GENI_RX_FIFOn, &fifo_word, 1); + for (j = 0; j < bytes_to_read; j++) + rx_buf[i++] = fifo_byte[j]; + } + mas->rx_rem_bytes -= rx_bytes; +} + +static irqreturn_t geni_spi_isr(int irq, void *data) +{ + struct spi_master *spi = data; + struct spi_geni_master *mas = spi_master_get_devdata(spi); + struct geni_se *se = &mas->se; + u32 m_irq; + unsigned long flags; + irqreturn_t ret = IRQ_HANDLED; + + if (mas->cur_mcmd == CMD_NONE) + return IRQ_NONE; + + spin_lock_irqsave(&mas->lock, flags); + m_irq = readl(se->base + SE_GENI_M_IRQ_STATUS); + + if ((m_irq & M_RX_FIFO_WATERMARK_EN) || (m_irq & M_RX_FIFO_LAST_EN)) + geni_spi_handle_rx(mas); + + if (m_irq & M_TX_FIFO_WATERMARK_EN) + geni_spi_handle_tx(mas); + + if (m_irq & M_CMD_DONE_EN) { + if (mas->cur_mcmd == CMD_XFER) + spi_finalize_current_transfer(spi); + else if (mas->cur_mcmd == CMD_CS) + complete(&mas->xfer_done); + mas->cur_mcmd = CMD_NONE; + /* + * If this happens, then a CMD_DONE came before all the Tx + * buffer bytes were sent out. This is unusual, log this + * condition and disable the WM interrupt to prevent the + * system from stalling due an interrupt storm. + * If this happens when all Rx bytes haven't been received, log + * the condition. + * The only known time this can happen is if bits_per_word != 8 + * and some registers that expect xfer lengths in num spi_words + * weren't written correctly. + */ + if (mas->tx_rem_bytes) { + writel(0, se->base + SE_GENI_TX_WATERMARK_REG); + dev_err(mas->dev, "Premature done. tx_rem = %d bpw%d\n", + mas->tx_rem_bytes, mas->cur_bits_per_word); + } + if (mas->rx_rem_bytes) + dev_err(mas->dev, "Premature done. rx_rem = %d bpw%d\n", + mas->rx_rem_bytes, mas->cur_bits_per_word); + } + + if ((m_irq & M_CMD_CANCEL_EN) || (m_irq & M_CMD_ABORT_EN)) { + mas->cur_mcmd = CMD_NONE; + complete(&mas->xfer_done); + } + + writel(m_irq, se->base + SE_GENI_M_IRQ_CLEAR); + spin_unlock_irqrestore(&mas->lock, flags); + return ret; +} + +static int spi_geni_probe(struct platform_device *pdev) +{ + int ret; + struct spi_master *spi; + struct spi_geni_master *mas; + struct resource *res; + struct geni_se *se; + + spi = spi_alloc_master(&pdev->dev, sizeof(*mas)); + if (!spi) + return -ENOMEM; + + platform_set_drvdata(pdev, spi); + mas = spi_master_get_devdata(spi); + mas->dev = &pdev->dev; + mas->se.dev = &pdev->dev; + mas->se.wrapper = dev_get_drvdata(pdev->dev.parent); + se = &mas->se; + + spi->bus_num = -1; + spi->dev.of_node = pdev->dev.of_node; + mas->se.clk = devm_clk_get(&pdev->dev, "se"); + if (IS_ERR(mas->se.clk)) { + ret = PTR_ERR(mas->se.clk); + dev_err(&pdev->dev, "Err getting SE Core clk %d\n", ret); + goto spi_geni_probe_err; + } + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + se->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(se->base)) { + ret = PTR_ERR(se->base); + goto spi_geni_probe_err; + } + + spi->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LOOP | SPI_CS_HIGH; + spi->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32); + spi->num_chipselect = 4; + spi->max_speed_hz = 50000000; + spi->prepare_message = spi_geni_prepare_message; + spi->transfer_one = spi_geni_transfer_one; + spi->auto_runtime_pm = true; + spi->handle_err = handle_fifo_timeout; + spi->set_cs = spi_geni_set_cs; + + init_completion(&mas->xfer_done); + spin_lock_init(&mas->lock); + pm_runtime_enable(&pdev->dev); + + ret = spi_geni_init(mas); + if (ret) + goto spi_geni_probe_runtime_disable; + + mas->irq = platform_get_irq(pdev, 0); + if (mas->irq < 0) { + ret = mas->irq; + dev_err(&pdev->dev, "Err getting IRQ %d\n", ret); + goto spi_geni_probe_runtime_disable; + } + + ret = request_irq(mas->irq, geni_spi_isr, + IRQF_TRIGGER_HIGH, "spi_geni", spi); + if (ret) + goto spi_geni_probe_runtime_disable; + + ret = spi_register_master(spi); + if (ret) + goto spi_geni_probe_free_irq; + + return 0; +spi_geni_probe_free_irq: + free_irq(mas->irq, spi); +spi_geni_probe_runtime_disable: + pm_runtime_disable(&pdev->dev); +spi_geni_probe_err: + spi_master_put(spi); + return ret; +} + +static int spi_geni_remove(struct platform_device *pdev) +{ + struct spi_master *spi = platform_get_drvdata(pdev); + struct spi_geni_master *mas = spi_master_get_devdata(spi); + + /* Unregister _before_ disabling pm_runtime() so we stop transfers */ + spi_unregister_master(spi); + + free_irq(mas->irq, spi); + pm_runtime_disable(&pdev->dev); + return 0; +} + +static int __maybe_unused spi_geni_runtime_suspend(struct device *dev) +{ + struct spi_master *spi = dev_get_drvdata(dev); + struct spi_geni_master *mas = spi_master_get_devdata(spi); + + return geni_se_resources_off(&mas->se); +} + +static int __maybe_unused spi_geni_runtime_resume(struct device *dev) +{ + struct spi_master *spi = dev_get_drvdata(dev); + struct spi_geni_master *mas = spi_master_get_devdata(spi); + + return geni_se_resources_on(&mas->se); +} + +static int __maybe_unused spi_geni_suspend(struct device *dev) +{ + struct spi_master *spi = dev_get_drvdata(dev); + int ret; + + ret = spi_master_suspend(spi); + if (ret) + return ret; + + ret = pm_runtime_force_suspend(dev); + if (ret) + spi_master_resume(spi); + + return ret; +} + +static int __maybe_unused spi_geni_resume(struct device *dev) +{ + struct spi_master *spi = dev_get_drvdata(dev); + int ret; + + ret = pm_runtime_force_resume(dev); + if (ret) + return ret; + + ret = spi_master_resume(spi); + if (ret) + pm_runtime_force_suspend(dev); + + return ret; +} + +static const struct dev_pm_ops spi_geni_pm_ops = { + SET_RUNTIME_PM_OPS(spi_geni_runtime_suspend, + spi_geni_runtime_resume, NULL) + SET_SYSTEM_SLEEP_PM_OPS(spi_geni_suspend, spi_geni_resume) +}; + +static const struct of_device_id spi_geni_dt_match[] = { + { .compatible = "qcom,geni-spi" }, + {} +}; +MODULE_DEVICE_TABLE(of, spi_geni_dt_match); + +static struct platform_driver spi_geni_driver = { + .probe = spi_geni_probe, + .remove = spi_geni_remove, + .driver = { + .name = "geni_spi", + .pm = &spi_geni_pm_ops, + .of_match_table = spi_geni_dt_match, + }, +}; +module_platform_driver(spi_geni_driver); + +MODULE_DESCRIPTION("SPI driver for GENI based QUP cores"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/spi/spi-gpio.c b/drivers/spi/spi-gpio.c index 421bfc7dda67..45973ee3ae11 100644 --- a/drivers/spi/spi-gpio.c +++ b/drivers/spi/spi-gpio.c @@ -295,9 +295,11 @@ static int spi_gpio_request(struct device *dev, spi_gpio->miso = devm_gpiod_get_optional(dev, "miso", GPIOD_IN); if (IS_ERR(spi_gpio->miso)) return PTR_ERR(spi_gpio->miso); - if (!spi_gpio->miso) - /* HW configuration without MISO pin */ - *mflags |= SPI_MASTER_NO_RX; + /* + * No setting SPI_MASTER_NO_RX here - if there is only a MOSI + * pin connected the host can still do RX by changing the + * direction of the line. + */ spi_gpio->sck = devm_gpiod_get(dev, "sck", GPIOD_OUT_LOW); if (IS_ERR(spi_gpio->sck)) @@ -423,7 +425,7 @@ static int spi_gpio_probe(struct platform_device *pdev) spi_gpio->bitbang.chipselect = spi_gpio_chipselect; spi_gpio->bitbang.set_line_direction = spi_gpio_set_direction; - if ((master_flags & (SPI_MASTER_NO_TX | SPI_MASTER_NO_RX)) == 0) { + if ((master_flags & SPI_MASTER_NO_TX) == 0) { spi_gpio->bitbang.txrx_word[SPI_MODE_0] = spi_gpio_txrx_word_mode0; spi_gpio->bitbang.txrx_word[SPI_MODE_1] = spi_gpio_txrx_word_mode1; spi_gpio->bitbang.txrx_word[SPI_MODE_2] = spi_gpio_txrx_word_mode2; @@ -447,10 +449,8 @@ static int spi_gpio_probe(struct platform_device *pdev) static int spi_gpio_remove(struct platform_device *pdev) { struct spi_gpio *spi_gpio; - struct spi_gpio_platform_data *pdata; spi_gpio = platform_get_drvdata(pdev); - pdata = dev_get_platdata(&pdev->dev); /* stop() unregisters child devices too */ spi_bitbang_stop(&spi_gpio->bitbang); diff --git a/drivers/spi/spi-imx.c b/drivers/spi/spi-imx.c index 08dd3a31a3e5..dd1ce12aa386 100644 --- a/drivers/spi/spi-imx.c +++ b/drivers/spi/spi-imx.c @@ -63,6 +63,7 @@ struct spi_imx_devtype_data { void (*trigger)(struct spi_imx_data *); int (*rx_available)(struct spi_imx_data *); void (*reset)(struct spi_imx_data *); + void (*setup_wml)(struct spi_imx_data *); void (*disable)(struct spi_imx_data *); bool has_dmamode; bool has_slavemode; @@ -216,7 +217,6 @@ static bool spi_imx_can_dma(struct spi_master *master, struct spi_device *spi, struct spi_transfer *transfer) { struct spi_imx_data *spi_imx = spi_master_get_devdata(master); - unsigned int bytes_per_word, i; if (!master->dma_rx) return false; @@ -224,14 +224,9 @@ static bool spi_imx_can_dma(struct spi_master *master, struct spi_device *spi, if (spi_imx->slave_mode) return false; - bytes_per_word = spi_imx_bytes_per_word(transfer->bits_per_word); - - for (i = spi_imx->devtype_data->fifo_size / 2; i > 0; i--) { - if (!(transfer->len % (i * bytes_per_word))) - break; - } + if (transfer->len < spi_imx->devtype_data->fifo_size) + return false; - spi_imx->wml = i; spi_imx->dynamic_burst = 0; return true; @@ -583,18 +578,21 @@ static int mx51_ecspi_config(struct spi_device *spi) else /* SCLK is _very_ slow */ usleep_range(delay, delay + 10); + return 0; +} + +static void mx51_setup_wml(struct spi_imx_data *spi_imx) +{ /* * Configure the DMA register: setup the watermark * and enable DMA request. */ - writel(MX51_ECSPI_DMA_RX_WML(spi_imx->wml) | + writel(MX51_ECSPI_DMA_RX_WML(spi_imx->wml - 1) | MX51_ECSPI_DMA_TX_WML(spi_imx->wml) | MX51_ECSPI_DMA_RXT_WML(spi_imx->wml) | MX51_ECSPI_DMA_TEDEN | MX51_ECSPI_DMA_RXDEN | MX51_ECSPI_DMA_RXTDEN, spi_imx->base + MX51_ECSPI_DMA); - - return 0; } static int mx51_ecspi_rx_available(struct spi_imx_data *spi_imx) @@ -931,6 +929,7 @@ static struct spi_imx_devtype_data imx51_ecspi_devtype_data = { .trigger = mx51_ecspi_trigger, .rx_available = mx51_ecspi_rx_available, .reset = mx51_ecspi_reset, + .setup_wml = mx51_setup_wml, .fifo_size = 64, .has_dmamode = true, .dynamic_burst = true, @@ -1138,7 +1137,6 @@ static int spi_imx_setupxfer(struct spi_device *spi, struct spi_transfer *t) { struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master); - int ret; if (!t) return 0; @@ -1179,12 +1177,6 @@ static int spi_imx_setupxfer(struct spi_device *spi, else spi_imx->usedma = 0; - if (spi_imx->usedma) { - ret = spi_imx_dma_configure(spi->master); - if (ret) - return ret; - } - if (is_imx53_ecspi(spi_imx) && spi_imx->slave_mode) { spi_imx->rx = mx53_ecspi_rx_slave; spi_imx->tx = mx53_ecspi_tx_slave; @@ -1289,6 +1281,31 @@ static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx, unsigned long timeout; struct spi_master *master = spi_imx->bitbang.master; struct sg_table *tx = &transfer->tx_sg, *rx = &transfer->rx_sg; + struct scatterlist *last_sg = sg_last(rx->sgl, rx->nents); + unsigned int bytes_per_word, i; + int ret; + + /* Get the right burst length from the last sg to ensure no tail data */ + bytes_per_word = spi_imx_bytes_per_word(transfer->bits_per_word); + for (i = spi_imx->devtype_data->fifo_size / 2; i > 0; i--) { + if (!(sg_dma_len(last_sg) % (i * bytes_per_word))) + break; + } + /* Use 1 as wml in case no available burst length got */ + if (i == 0) + i = 1; + + spi_imx->wml = i; + + ret = spi_imx_dma_configure(master); + if (ret) + return ret; + + if (!spi_imx->devtype_data->setup_wml) { + dev_err(spi_imx->dev, "No setup_wml()?\n"); + return -EINVAL; + } + spi_imx->devtype_data->setup_wml(spi_imx); /* * The TX DMA setup starts the transfer, so make sure RX is configured diff --git a/drivers/spi/spi-mem.c b/drivers/spi/spi-mem.c index e43842c7a31a..62a7b80801d2 100644 --- a/drivers/spi/spi-mem.c +++ b/drivers/spi/spi-mem.c @@ -12,6 +12,8 @@ #include "internals.h" +#define SPI_MEM_MAX_BUSWIDTH 4 + /** * spi_controller_dma_map_mem_op_data() - DMA-map the buffer attached to a * memory operation @@ -149,6 +151,44 @@ static bool spi_mem_default_supports_op(struct spi_mem *mem, } EXPORT_SYMBOL_GPL(spi_mem_default_supports_op); +static bool spi_mem_buswidth_is_valid(u8 buswidth) +{ + if (hweight8(buswidth) > 1 || buswidth > SPI_MEM_MAX_BUSWIDTH) + return false; + + return true; +} + +static int spi_mem_check_op(const struct spi_mem_op *op) +{ + if (!op->cmd.buswidth) + return -EINVAL; + + if ((op->addr.nbytes && !op->addr.buswidth) || + (op->dummy.nbytes && !op->dummy.buswidth) || + (op->data.nbytes && !op->data.buswidth)) + return -EINVAL; + + if (!spi_mem_buswidth_is_valid(op->cmd.buswidth) || + !spi_mem_buswidth_is_valid(op->addr.buswidth) || + !spi_mem_buswidth_is_valid(op->dummy.buswidth) || + !spi_mem_buswidth_is_valid(op->data.buswidth)) + return -EINVAL; + + return 0; +} + +static bool spi_mem_internal_supports_op(struct spi_mem *mem, + const struct spi_mem_op *op) +{ + struct spi_controller *ctlr = mem->spi->controller; + + if (ctlr->mem_ops && ctlr->mem_ops->supports_op) + return ctlr->mem_ops->supports_op(mem, op); + + return spi_mem_default_supports_op(mem, op); +} + /** * spi_mem_supports_op() - Check if a memory device and the controller it is * connected to support a specific memory operation @@ -166,12 +206,10 @@ EXPORT_SYMBOL_GPL(spi_mem_default_supports_op); */ bool spi_mem_supports_op(struct spi_mem *mem, const struct spi_mem_op *op) { - struct spi_controller *ctlr = mem->spi->controller; - - if (ctlr->mem_ops && ctlr->mem_ops->supports_op) - return ctlr->mem_ops->supports_op(mem, op); + if (spi_mem_check_op(op)) + return false; - return spi_mem_default_supports_op(mem, op); + return spi_mem_internal_supports_op(mem, op); } EXPORT_SYMBOL_GPL(spi_mem_supports_op); @@ -196,7 +234,11 @@ int spi_mem_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) u8 *tmpbuf; int ret; - if (!spi_mem_supports_op(mem, op)) + ret = spi_mem_check_op(op); + if (ret) + return ret; + + if (!spi_mem_internal_supports_op(mem, op)) return -ENOTSUPP; if (ctlr->mem_ops) { @@ -346,10 +388,25 @@ EXPORT_SYMBOL_GPL(spi_mem_get_name); int spi_mem_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op) { struct spi_controller *ctlr = mem->spi->controller; + size_t len; + + len = sizeof(op->cmd.opcode) + op->addr.nbytes + op->dummy.nbytes; if (ctlr->mem_ops && ctlr->mem_ops->adjust_op_size) return ctlr->mem_ops->adjust_op_size(mem, op); + if (!ctlr->mem_ops || !ctlr->mem_ops->exec_op) { + if (len > spi_max_transfer_size(mem->spi)) + return -EINVAL; + + op->data.nbytes = min3((size_t)op->data.nbytes, + spi_max_transfer_size(mem->spi), + spi_max_message_size(mem->spi) - + len); + if (!op->data.nbytes) + return -EINVAL; + } + return 0; } EXPORT_SYMBOL_GPL(spi_mem_adjust_op_size); diff --git a/drivers/spi/spi-mt65xx.c b/drivers/spi/spi-mt65xx.c index 86bf45667a04..3dc31627c655 100644 --- a/drivers/spi/spi-mt65xx.c +++ b/drivers/spi/spi-mt65xx.c @@ -98,6 +98,7 @@ struct mtk_spi { struct clk *parent_clk, *sel_clk, *spi_clk; struct spi_transfer *cur_transfer; u32 xfer_len; + u32 num_xfered; struct scatterlist *tx_sgl, *rx_sgl; u32 tx_sgl_len, rx_sgl_len; const struct mtk_spi_compatible *dev_comp; @@ -385,6 +386,7 @@ static int mtk_spi_fifo_transfer(struct spi_master *master, mdata->cur_transfer = xfer; mdata->xfer_len = min(MTK_SPI_MAX_FIFO_SIZE, xfer->len); + mdata->num_xfered = 0; mtk_spi_prepare_transfer(master, xfer); mtk_spi_setup_packet(master); @@ -415,6 +417,7 @@ static int mtk_spi_dma_transfer(struct spi_master *master, mdata->tx_sgl_len = 0; mdata->rx_sgl_len = 0; mdata->cur_transfer = xfer; + mdata->num_xfered = 0; mtk_spi_prepare_transfer(master, xfer); @@ -482,7 +485,7 @@ static int mtk_spi_setup(struct spi_device *spi) static irqreturn_t mtk_spi_interrupt(int irq, void *dev_id) { - u32 cmd, reg_val, cnt, remainder; + u32 cmd, reg_val, cnt, remainder, len; struct spi_master *master = dev_id; struct mtk_spi *mdata = spi_master_get_devdata(master); struct spi_transfer *trans = mdata->cur_transfer; @@ -497,36 +500,38 @@ static irqreturn_t mtk_spi_interrupt(int irq, void *dev_id) if (trans->rx_buf) { cnt = mdata->xfer_len / 4; ioread32_rep(mdata->base + SPI_RX_DATA_REG, - trans->rx_buf, cnt); + trans->rx_buf + mdata->num_xfered, cnt); remainder = mdata->xfer_len % 4; if (remainder > 0) { reg_val = readl(mdata->base + SPI_RX_DATA_REG); - memcpy(trans->rx_buf + (cnt * 4), - ®_val, remainder); + memcpy(trans->rx_buf + + mdata->num_xfered + + (cnt * 4), + ®_val, + remainder); } } - trans->len -= mdata->xfer_len; - if (!trans->len) { + mdata->num_xfered += mdata->xfer_len; + if (mdata->num_xfered == trans->len) { spi_finalize_current_transfer(master); return IRQ_HANDLED; } - if (trans->tx_buf) - trans->tx_buf += mdata->xfer_len; - if (trans->rx_buf) - trans->rx_buf += mdata->xfer_len; - - mdata->xfer_len = min(MTK_SPI_MAX_FIFO_SIZE, trans->len); + len = trans->len - mdata->num_xfered; + mdata->xfer_len = min(MTK_SPI_MAX_FIFO_SIZE, len); mtk_spi_setup_packet(master); - cnt = trans->len / 4; - iowrite32_rep(mdata->base + SPI_TX_DATA_REG, trans->tx_buf, cnt); + cnt = len / 4; + iowrite32_rep(mdata->base + SPI_TX_DATA_REG, + trans->tx_buf + mdata->num_xfered, cnt); - remainder = trans->len % 4; + remainder = len % 4; if (remainder > 0) { reg_val = 0; - memcpy(®_val, trans->tx_buf + (cnt * 4), remainder); + memcpy(®_val, + trans->tx_buf + (cnt * 4) + mdata->num_xfered, + remainder); writel(reg_val, mdata->base + SPI_TX_DATA_REG); } diff --git a/drivers/spi/spi-omap2-mcspi.c b/drivers/spi/spi-omap2-mcspi.c index 508c61c669e7..f024c3fc3679 100644 --- a/drivers/spi/spi-omap2-mcspi.c +++ b/drivers/spi/spi-omap2-mcspi.c @@ -33,6 +33,7 @@ #include <linux/of.h> #include <linux/of_device.h> #include <linux/gcd.h> +#include <linux/iopoll.h> #include <linux/spi/spi.h> #include <linux/gpio.h> @@ -126,6 +127,7 @@ struct omap2_mcspi_regs { }; struct omap2_mcspi { + struct completion txdone; struct spi_master *master; /* Virtual base address of the controller */ void __iomem *base; @@ -135,6 +137,7 @@ struct omap2_mcspi { struct device *dev; struct omap2_mcspi_regs ctx; int fifo_depth; + bool slave_aborted; unsigned int pin_dir:1; }; @@ -274,19 +277,23 @@ static void omap2_mcspi_set_cs(struct spi_device *spi, bool enable) } } -static void omap2_mcspi_set_master_mode(struct spi_master *master) +static void omap2_mcspi_set_mode(struct spi_master *master) { struct omap2_mcspi *mcspi = spi_master_get_devdata(master); struct omap2_mcspi_regs *ctx = &mcspi->ctx; u32 l; /* - * Setup when switching from (reset default) slave mode - * to single-channel master mode + * Choose master or slave mode */ l = mcspi_read_reg(master, OMAP2_MCSPI_MODULCTRL); - l &= ~(OMAP2_MCSPI_MODULCTRL_STEST | OMAP2_MCSPI_MODULCTRL_MS); - l |= OMAP2_MCSPI_MODULCTRL_SINGLE; + l &= ~(OMAP2_MCSPI_MODULCTRL_STEST); + if (spi_controller_is_slave(master)) { + l |= (OMAP2_MCSPI_MODULCTRL_MS); + } else { + l &= ~(OMAP2_MCSPI_MODULCTRL_MS); + l |= OMAP2_MCSPI_MODULCTRL_SINGLE; + } mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, l); ctx->modulctrl = l; @@ -299,7 +306,7 @@ static void omap2_mcspi_set_fifo(const struct spi_device *spi, struct omap2_mcspi_cs *cs = spi->controller_state; struct omap2_mcspi *mcspi; unsigned int wcnt; - int max_fifo_depth, fifo_depth, bytes_per_word; + int max_fifo_depth, bytes_per_word; u32 chconf, xferlevel; mcspi = spi_master_get_devdata(master); @@ -315,10 +322,6 @@ static void omap2_mcspi_set_fifo(const struct spi_device *spi, else max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH; - fifo_depth = gcd(t->len, max_fifo_depth); - if (fifo_depth < 2 || fifo_depth % bytes_per_word != 0) - goto disable_fifo; - wcnt = t->len / bytes_per_word; if (wcnt > OMAP2_MCSPI_MAX_FIFOWCNT) goto disable_fifo; @@ -326,16 +329,17 @@ static void omap2_mcspi_set_fifo(const struct spi_device *spi, xferlevel = wcnt << 16; if (t->rx_buf != NULL) { chconf |= OMAP2_MCSPI_CHCONF_FFER; - xferlevel |= (fifo_depth - 1) << 8; + xferlevel |= (bytes_per_word - 1) << 8; } + if (t->tx_buf != NULL) { chconf |= OMAP2_MCSPI_CHCONF_FFET; - xferlevel |= fifo_depth - 1; + xferlevel |= bytes_per_word - 1; } mcspi_write_reg(master, OMAP2_MCSPI_XFERLEVEL, xferlevel); mcspi_write_chconf0(spi, chconf); - mcspi->fifo_depth = fifo_depth; + mcspi->fifo_depth = max_fifo_depth; return; } @@ -353,18 +357,22 @@ disable_fifo: static int mcspi_wait_for_reg_bit(void __iomem *reg, unsigned long bit) { - unsigned long timeout; - - timeout = jiffies + msecs_to_jiffies(1000); - while (!(readl_relaxed(reg) & bit)) { - if (time_after(jiffies, timeout)) { - if (!(readl_relaxed(reg) & bit)) - return -ETIMEDOUT; - else - return 0; - } - cpu_relax(); + u32 val; + + return readl_poll_timeout(reg, val, val & bit, 1, MSEC_PER_SEC); +} + +static int mcspi_wait_for_completion(struct omap2_mcspi *mcspi, + struct completion *x) +{ + if (spi_controller_is_slave(mcspi->master)) { + if (wait_for_completion_interruptible(x) || + mcspi->slave_aborted) + return -EINTR; + } else { + wait_for_completion(x); } + return 0; } @@ -517,7 +525,12 @@ omap2_mcspi_rx_dma(struct spi_device *spi, struct spi_transfer *xfer, dma_async_issue_pending(mcspi_dma->dma_rx); omap2_mcspi_set_dma_req(spi, 1, 1); - wait_for_completion(&mcspi_dma->dma_rx_completion); + ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_rx_completion); + if (ret || mcspi->slave_aborted) { + dmaengine_terminate_sync(mcspi_dma->dma_rx); + omap2_mcspi_set_dma_req(spi, 1, 0); + return 0; + } for (x = 0; x < nb_sizes; x++) kfree(sg_out[x]); @@ -585,7 +598,6 @@ omap2_mcspi_txrx_dma(struct spi_device *spi, struct spi_transfer *xfer) struct dma_slave_config cfg; enum dma_slave_buswidth width; unsigned es; - u32 burst; void __iomem *chstat_reg; void __iomem *irqstat_reg; int wait_res; @@ -605,34 +617,49 @@ omap2_mcspi_txrx_dma(struct spi_device *spi, struct spi_transfer *xfer) } count = xfer->len; - burst = 1; - - if (mcspi->fifo_depth > 0) { - if (count > mcspi->fifo_depth) - burst = mcspi->fifo_depth / es; - else - burst = count / es; - } memset(&cfg, 0, sizeof(cfg)); cfg.src_addr = cs->phys + OMAP2_MCSPI_RX0; cfg.dst_addr = cs->phys + OMAP2_MCSPI_TX0; cfg.src_addr_width = width; cfg.dst_addr_width = width; - cfg.src_maxburst = burst; - cfg.dst_maxburst = burst; + cfg.src_maxburst = es; + cfg.dst_maxburst = es; rx = xfer->rx_buf; tx = xfer->tx_buf; - if (tx != NULL) + mcspi->slave_aborted = false; + reinit_completion(&mcspi_dma->dma_tx_completion); + reinit_completion(&mcspi_dma->dma_rx_completion); + reinit_completion(&mcspi->txdone); + if (tx) { + /* Enable EOW IRQ to know end of tx in slave mode */ + if (spi_controller_is_slave(spi->master)) + mcspi_write_reg(spi->master, + OMAP2_MCSPI_IRQENABLE, + OMAP2_MCSPI_IRQSTATUS_EOW); omap2_mcspi_tx_dma(spi, xfer, cfg); + } if (rx != NULL) count = omap2_mcspi_rx_dma(spi, xfer, cfg, es); if (tx != NULL) { - wait_for_completion(&mcspi_dma->dma_tx_completion); + int ret; + + ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_tx_completion); + if (ret || mcspi->slave_aborted) { + dmaengine_terminate_sync(mcspi_dma->dma_tx); + omap2_mcspi_set_dma_req(spi, 0, 0); + return 0; + } + + if (spi_controller_is_slave(mcspi->master)) { + ret = mcspi_wait_for_completion(mcspi, &mcspi->txdone); + if (ret || mcspi->slave_aborted) + return 0; + } if (mcspi->fifo_depth > 0) { irqstat_reg = mcspi->base + OMAP2_MCSPI_IRQSTATUS; @@ -1089,6 +1116,36 @@ static void omap2_mcspi_cleanup(struct spi_device *spi) gpio_free(spi->cs_gpio); } +static irqreturn_t omap2_mcspi_irq_handler(int irq, void *data) +{ + struct omap2_mcspi *mcspi = data; + u32 irqstat; + + irqstat = mcspi_read_reg(mcspi->master, OMAP2_MCSPI_IRQSTATUS); + if (!irqstat) + return IRQ_NONE; + + /* Disable IRQ and wakeup slave xfer task */ + mcspi_write_reg(mcspi->master, OMAP2_MCSPI_IRQENABLE, 0); + if (irqstat & OMAP2_MCSPI_IRQSTATUS_EOW) + complete(&mcspi->txdone); + + return IRQ_HANDLED; +} + +static int omap2_mcspi_slave_abort(struct spi_master *master) +{ + struct omap2_mcspi *mcspi = spi_master_get_devdata(master); + struct omap2_mcspi_dma *mcspi_dma = mcspi->dma_channels; + + mcspi->slave_aborted = true; + complete(&mcspi_dma->dma_rx_completion); + complete(&mcspi_dma->dma_tx_completion); + complete(&mcspi->txdone); + + return 0; +} + static int omap2_mcspi_transfer_one(struct spi_master *master, struct spi_device *spi, struct spi_transfer *t) @@ -1255,10 +1312,20 @@ static bool omap2_mcspi_can_dma(struct spi_master *master, struct spi_device *spi, struct spi_transfer *xfer) { + struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master); + struct omap2_mcspi_dma *mcspi_dma = + &mcspi->dma_channels[spi->chip_select]; + + if (!mcspi_dma->dma_rx || !mcspi_dma->dma_tx) + return false; + + if (spi_controller_is_slave(master)) + return true; + return (xfer->len >= DMA_MIN_BYTES); } -static int omap2_mcspi_master_setup(struct omap2_mcspi *mcspi) +static int omap2_mcspi_controller_setup(struct omap2_mcspi *mcspi) { struct spi_master *master = mcspi->master; struct omap2_mcspi_regs *ctx = &mcspi->ctx; @@ -1275,7 +1342,7 @@ static int omap2_mcspi_master_setup(struct omap2_mcspi *mcspi) OMAP2_MCSPI_WAKEUPENABLE_WKEN); ctx->wakeupenable = OMAP2_MCSPI_WAKEUPENABLE_WKEN; - omap2_mcspi_set_master_mode(master); + omap2_mcspi_set_mode(master); pm_runtime_mark_last_busy(mcspi->dev); pm_runtime_put_autosuspend(mcspi->dev); return 0; @@ -1350,11 +1417,12 @@ static int omap2_mcspi_probe(struct platform_device *pdev) struct device_node *node = pdev->dev.of_node; const struct of_device_id *match; - master = spi_alloc_master(&pdev->dev, sizeof *mcspi); - if (master == NULL) { - dev_dbg(&pdev->dev, "master allocation failed\n"); + if (of_property_read_bool(node, "spi-slave")) + master = spi_alloc_slave(&pdev->dev, sizeof(*mcspi)); + else + master = spi_alloc_master(&pdev->dev, sizeof(*mcspi)); + if (!master) return -ENOMEM; - } /* the spi->mode bits understood by this driver: */ master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH; @@ -1366,6 +1434,7 @@ static int omap2_mcspi_probe(struct platform_device *pdev) master->transfer_one = omap2_mcspi_transfer_one; master->set_cs = omap2_mcspi_set_cs; master->cleanup = omap2_mcspi_cleanup; + master->slave_abort = omap2_mcspi_slave_abort; master->dev.of_node = node; master->max_speed_hz = OMAP2_MCSPI_MAX_FREQ; master->min_speed_hz = OMAP2_MCSPI_MAX_FREQ >> 15; @@ -1417,15 +1486,31 @@ static int omap2_mcspi_probe(struct platform_device *pdev) sprintf(mcspi->dma_channels[i].dma_tx_ch_name, "tx%d", i); } + status = platform_get_irq(pdev, 0); + if (status == -EPROBE_DEFER) + goto free_master; + if (status < 0) { + dev_err(&pdev->dev, "no irq resource found\n"); + goto free_master; + } + init_completion(&mcspi->txdone); + status = devm_request_irq(&pdev->dev, status, + omap2_mcspi_irq_handler, 0, pdev->name, + mcspi); + if (status) { + dev_err(&pdev->dev, "Cannot request IRQ"); + goto free_master; + } + pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_set_autosuspend_delay(&pdev->dev, SPI_AUTOSUSPEND_TIMEOUT); pm_runtime_enable(&pdev->dev); - status = omap2_mcspi_master_setup(mcspi); + status = omap2_mcspi_controller_setup(mcspi); if (status < 0) goto disable_pm; - status = devm_spi_register_master(&pdev->dev, master); + status = devm_spi_register_controller(&pdev->dev, master); if (status < 0) goto disable_pm; diff --git a/drivers/spi/spi-orion.c b/drivers/spi/spi-orion.c index 47ef6b1a2e76..7f280567093e 100644 --- a/drivers/spi/spi-orion.c +++ b/drivers/spi/spi-orion.c @@ -431,6 +431,7 @@ orion_spi_write_read(struct spi_device *spi, struct spi_transfer *xfer) int word_len; struct orion_spi *orion_spi; int cs = spi->chip_select; + void __iomem *vaddr; word_len = spi->bits_per_word; count = xfer->len; @@ -441,8 +442,9 @@ orion_spi_write_read(struct spi_device *spi, struct spi_transfer *xfer) * Use SPI direct write mode if base address is available. Otherwise * fall back to PIO mode for this transfer. */ - if ((orion_spi->child[cs].direct_access.vaddr) && (xfer->tx_buf) && - (word_len == 8)) { + vaddr = orion_spi->child[cs].direct_access.vaddr; + + if (vaddr && xfer->tx_buf && word_len == 8) { unsigned int cnt = count / 4; unsigned int rem = count % 4; @@ -450,13 +452,11 @@ orion_spi_write_read(struct spi_device *spi, struct spi_transfer *xfer) * Send the TX-data to the SPI device via the direct * mapped address window */ - iowrite32_rep(orion_spi->child[cs].direct_access.vaddr, - xfer->tx_buf, cnt); + iowrite32_rep(vaddr, xfer->tx_buf, cnt); if (rem) { u32 *buf = (u32 *)xfer->tx_buf; - iowrite8_rep(orion_spi->child[cs].direct_access.vaddr, - &buf[cnt], rem); + iowrite8_rep(vaddr, &buf[cnt], rem); } return count; @@ -683,6 +683,7 @@ static int orion_spi_probe(struct platform_device *pdev) } for_each_available_child_of_node(pdev->dev.of_node, np) { + struct orion_direct_acc *dir_acc; u32 cs; int cs_gpio; @@ -750,14 +751,13 @@ static int orion_spi_probe(struct platform_device *pdev) * This needs to get extended for the direct SPI-NOR / SPI-NAND * support, once this gets implemented. */ - spi->child[cs].direct_access.vaddr = devm_ioremap(&pdev->dev, - r->start, - PAGE_SIZE); - if (!spi->child[cs].direct_access.vaddr) { + dir_acc = &spi->child[cs].direct_access; + dir_acc->vaddr = devm_ioremap(&pdev->dev, r->start, PAGE_SIZE); + if (!dir_acc->vaddr) { status = -ENOMEM; goto out_rel_axi_clk; } - spi->child[cs].direct_access.size = PAGE_SIZE; + dir_acc->size = PAGE_SIZE; dev_info(&pdev->dev, "CS%d configured for direct access\n", cs); } diff --git a/drivers/spi/spi-pic32-sqi.c b/drivers/spi/spi-pic32-sqi.c index bd1c6b53283f..d7e4e18ec3df 100644 --- a/drivers/spi/spi-pic32-sqi.c +++ b/drivers/spi/spi-pic32-sqi.c @@ -468,7 +468,7 @@ static int ring_desc_ring_alloc(struct pic32_sqi *sqi) /* allocate coherent DMAable memory for hardware buffer descriptors. */ sqi->bd = dma_zalloc_coherent(&sqi->master->dev, sizeof(*bd) * PESQI_BD_COUNT, - &sqi->bd_dma, GFP_DMA32); + &sqi->bd_dma, GFP_KERNEL); if (!sqi->bd) { dev_err(&sqi->master->dev, "failed allocating dma buffer\n"); return -ENOMEM; @@ -656,7 +656,7 @@ static int pic32_sqi_probe(struct platform_device *pdev) master->max_speed_hz = clk_get_rate(sqi->base_clk); master->dma_alignment = 32; master->max_dma_len = PESQI_BD_BUF_LEN_MAX; - master->dev.of_node = of_node_get(pdev->dev.of_node); + master->dev.of_node = pdev->dev.of_node; master->mode_bits = SPI_MODE_3 | SPI_MODE_0 | SPI_TX_DUAL | SPI_RX_DUAL | SPI_TX_QUAD | SPI_RX_QUAD; master->flags = SPI_MASTER_HALF_DUPLEX; diff --git a/drivers/spi/spi-pic32.c b/drivers/spi/spi-pic32.c index f8a45af1fa9f..131849adc570 100644 --- a/drivers/spi/spi-pic32.c +++ b/drivers/spi/spi-pic32.c @@ -320,7 +320,7 @@ static int pic32_spi_dma_transfer(struct pic32_spi *pic32s, desc_rx = dmaengine_prep_slave_sg(master->dma_rx, xfer->rx_sg.sgl, xfer->rx_sg.nents, - DMA_FROM_DEVICE, + DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); if (!desc_rx) { ret = -EINVAL; @@ -330,7 +330,7 @@ static int pic32_spi_dma_transfer(struct pic32_spi *pic32s, desc_tx = dmaengine_prep_slave_sg(master->dma_tx, xfer->tx_sg.sgl, xfer->tx_sg.nents, - DMA_TO_DEVICE, + DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); if (!desc_tx) { ret = -EINVAL; @@ -774,7 +774,7 @@ static int pic32_spi_probe(struct platform_device *pdev) if (ret) goto err_master; - master->dev.of_node = of_node_get(pdev->dev.of_node); + master->dev.of_node = pdev->dev.of_node; master->mode_bits = SPI_MODE_3 | SPI_MODE_0 | SPI_CS_HIGH; master->num_chipselect = 1; /* single chip-select */ master->max_speed_hz = clk_get_rate(pic32s->clk); diff --git a/drivers/spi/spi-pl022.c b/drivers/spi/spi-pl022.c index 1af8c96b940e..6120e6abcd96 100644 --- a/drivers/spi/spi-pl022.c +++ b/drivers/spi/spi-pl022.c @@ -1490,10 +1490,8 @@ static void do_polling_transfer(struct pl022 *pl022) struct spi_message *message = NULL; struct spi_transfer *transfer = NULL; struct spi_transfer *previous = NULL; - struct chip_data *chip; unsigned long time, timeout; - chip = pl022->cur_chip; message = pl022->cur_msg; while (message->state != STATE_DONE) { @@ -2325,10 +2323,8 @@ static int pl022_suspend(struct device *dev) int ret; ret = spi_master_suspend(pl022->master); - if (ret) { - dev_warn(dev, "cannot suspend master\n"); + if (ret) return ret; - } ret = pm_runtime_force_suspend(dev); if (ret) { @@ -2353,9 +2349,7 @@ static int pl022_resume(struct device *dev) /* Start the queue running */ ret = spi_master_resume(pl022->master); - if (ret) - dev_err(dev, "problem starting queue (%d)\n", ret); - else + if (!ret) dev_dbg(dev, "resumed\n"); return ret; diff --git a/drivers/spi/spi-pxa2xx.c b/drivers/spi/spi-pxa2xx.c index 14f4ea59caff..612cc49db28f 100644 --- a/drivers/spi/spi-pxa2xx.c +++ b/drivers/spi/spi-pxa2xx.c @@ -33,6 +33,7 @@ #include <linux/clk.h> #include <linux/pm_runtime.h> #include <linux/acpi.h> +#include <linux/of_device.h> #include "spi-pxa2xx.h" @@ -665,9 +666,11 @@ static irqreturn_t interrupt_transfer(struct driver_data *drv_data) bytes_left = drv_data->rx_end - drv_data->rx; switch (drv_data->n_bytes) { case 4: - bytes_left >>= 1; + bytes_left >>= 2; + break; case 2: bytes_left >>= 1; + break; } rx_thre = pxa2xx_spi_get_rx_default_thre(drv_data); @@ -1333,9 +1336,6 @@ static void cleanup(struct spi_device *spi) kfree(chip); } -#ifdef CONFIG_PCI -#ifdef CONFIG_ACPI - static const struct acpi_device_id pxa2xx_spi_acpi_match[] = { { "INT33C0", LPSS_LPT_SSP }, { "INT33C1", LPSS_LPT_SSP }, @@ -1347,23 +1347,6 @@ static const struct acpi_device_id pxa2xx_spi_acpi_match[] = { }; MODULE_DEVICE_TABLE(acpi, pxa2xx_spi_acpi_match); -static int pxa2xx_spi_get_port_id(struct acpi_device *adev) -{ - unsigned int devid; - int port_id = -1; - - if (adev && adev->pnp.unique_id && - !kstrtouint(adev->pnp.unique_id, 0, &devid)) - port_id = devid; - return port_id; -} -#else /* !CONFIG_ACPI */ -static int pxa2xx_spi_get_port_id(struct acpi_device *adev) -{ - return -1; -} -#endif - /* * PCI IDs of compound devices that integrate both host controller and private * integrated DMA engine. Please note these are not used in module @@ -1410,6 +1393,37 @@ static const struct pci_device_id pxa2xx_spi_pci_compound_match[] = { { }, }; +static const struct of_device_id pxa2xx_spi_of_match[] = { + { .compatible = "marvell,mmp2-ssp", .data = (void *)MMP2_SSP }, + {}, +}; +MODULE_DEVICE_TABLE(of, pxa2xx_spi_of_match); + +#ifdef CONFIG_ACPI + +static int pxa2xx_spi_get_port_id(struct acpi_device *adev) +{ + unsigned int devid; + int port_id = -1; + + if (adev && adev->pnp.unique_id && + !kstrtouint(adev->pnp.unique_id, 0, &devid)) + port_id = devid; + return port_id; +} + +#else /* !CONFIG_ACPI */ + +static int pxa2xx_spi_get_port_id(struct acpi_device *adev) +{ + return -1; +} + +#endif /* CONFIG_ACPI */ + + +#ifdef CONFIG_PCI + static bool pxa2xx_spi_idma_filter(struct dma_chan *chan, void *param) { struct device *dev = param; @@ -1420,6 +1434,8 @@ static bool pxa2xx_spi_idma_filter(struct dma_chan *chan, void *param) return true; } +#endif /* CONFIG_PCI */ + static struct pxa2xx_spi_master * pxa2xx_spi_init_pdata(struct platform_device *pdev) { @@ -1429,11 +1445,15 @@ pxa2xx_spi_init_pdata(struct platform_device *pdev) struct resource *res; const struct acpi_device_id *adev_id = NULL; const struct pci_device_id *pcidev_id = NULL; - int type; + const struct of_device_id *of_id = NULL; + enum pxa_ssp_type type; adev = ACPI_COMPANION(&pdev->dev); - if (dev_is_pci(pdev->dev.parent)) + if (pdev->dev.of_node) + of_id = of_match_device(pdev->dev.driver->of_match_table, + &pdev->dev); + else if (dev_is_pci(pdev->dev.parent)) pcidev_id = pci_match_id(pxa2xx_spi_pci_compound_match, to_pci_dev(pdev->dev.parent)); else if (adev) @@ -1443,9 +1463,11 @@ pxa2xx_spi_init_pdata(struct platform_device *pdev) return NULL; if (adev_id) - type = (int)adev_id->driver_data; + type = (enum pxa_ssp_type)adev_id->driver_data; else if (pcidev_id) - type = (int)pcidev_id->driver_data; + type = (enum pxa_ssp_type)pcidev_id->driver_data; + else if (of_id) + type = (enum pxa_ssp_type)of_id->data; else return NULL; @@ -1464,11 +1486,13 @@ pxa2xx_spi_init_pdata(struct platform_device *pdev) if (IS_ERR(ssp->mmio_base)) return NULL; +#ifdef CONFIG_PCI if (pcidev_id) { pdata->tx_param = pdev->dev.parent; pdata->rx_param = pdev->dev.parent; pdata->dma_filter = pxa2xx_spi_idma_filter; } +#endif ssp->clk = devm_clk_get(&pdev->dev, NULL); ssp->irq = platform_get_irq(pdev, 0); @@ -1482,14 +1506,6 @@ pxa2xx_spi_init_pdata(struct platform_device *pdev) return pdata; } -#else /* !CONFIG_PCI */ -static inline struct pxa2xx_spi_master * -pxa2xx_spi_init_pdata(struct platform_device *pdev) -{ - return NULL; -} -#endif - static int pxa2xx_spi_fw_translate_cs(struct spi_controller *master, unsigned int cs) { @@ -1764,14 +1780,6 @@ static int pxa2xx_spi_remove(struct platform_device *pdev) return 0; } -static void pxa2xx_spi_shutdown(struct platform_device *pdev) -{ - int status = 0; - - if ((status = pxa2xx_spi_remove(pdev)) != 0) - dev_err(&pdev->dev, "shutdown failed with %d\n", status); -} - #ifdef CONFIG_PM_SLEEP static int pxa2xx_spi_suspend(struct device *dev) { @@ -1808,13 +1816,7 @@ static int pxa2xx_spi_resume(struct device *dev) lpss_ssp_setup(drv_data); /* Start the queue running */ - status = spi_controller_resume(drv_data->master); - if (status != 0) { - dev_err(dev, "problem starting queue (%d)\n", status); - return status; - } - - return 0; + return spi_controller_resume(drv_data->master); } #endif @@ -1848,10 +1850,10 @@ static struct platform_driver driver = { .name = "pxa2xx-spi", .pm = &pxa2xx_spi_pm_ops, .acpi_match_table = ACPI_PTR(pxa2xx_spi_acpi_match), + .of_match_table = of_match_ptr(pxa2xx_spi_of_match), }, .probe = pxa2xx_spi_probe, .remove = pxa2xx_spi_remove, - .shutdown = pxa2xx_spi_shutdown, }; static int __init pxa2xx_spi_init(void) diff --git a/drivers/spi/spi-qcom-qspi.c b/drivers/spi/spi-qcom-qspi.c new file mode 100644 index 000000000000..b8163b40bb92 --- /dev/null +++ b/drivers/spi/spi-qcom-qspi.c @@ -0,0 +1,581 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2017-2018, The Linux foundation. All rights reserved. + +#include <linux/clk.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <linux/pm_runtime.h> +#include <linux/spi/spi.h> +#include <linux/spi/spi-mem.h> + + +#define QSPI_NUM_CS 2 +#define QSPI_BYTES_PER_WORD 4 + +#define MSTR_CONFIG 0x0000 +#define FULL_CYCLE_MODE BIT(3) +#define FB_CLK_EN BIT(4) +#define PIN_HOLDN BIT(6) +#define PIN_WPN BIT(7) +#define DMA_ENABLE BIT(8) +#define BIG_ENDIAN_MODE BIT(9) +#define SPI_MODE_MSK 0xc00 +#define SPI_MODE_SHFT 10 +#define CHIP_SELECT_NUM BIT(12) +#define SBL_EN BIT(13) +#define LPA_BASE_MSK 0x3c000 +#define LPA_BASE_SHFT 14 +#define TX_DATA_DELAY_MSK 0xc0000 +#define TX_DATA_DELAY_SHFT 18 +#define TX_CLK_DELAY_MSK 0x300000 +#define TX_CLK_DELAY_SHFT 20 +#define TX_CS_N_DELAY_MSK 0xc00000 +#define TX_CS_N_DELAY_SHFT 22 +#define TX_DATA_OE_DELAY_MSK 0x3000000 +#define TX_DATA_OE_DELAY_SHFT 24 + +#define AHB_MASTER_CFG 0x0004 +#define HMEM_TYPE_START_MID_TRANS_MSK 0x7 +#define HMEM_TYPE_START_MID_TRANS_SHFT 0 +#define HMEM_TYPE_LAST_TRANS_MSK 0x38 +#define HMEM_TYPE_LAST_TRANS_SHFT 3 +#define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_MSK 0xc0 +#define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_SHFT 6 +#define HMEMTYPE_READ_TRANS_MSK 0x700 +#define HMEMTYPE_READ_TRANS_SHFT 8 +#define HSHARED BIT(11) +#define HINNERSHARED BIT(12) + +#define MSTR_INT_EN 0x000C +#define MSTR_INT_STATUS 0x0010 +#define RESP_FIFO_UNDERRUN BIT(0) +#define RESP_FIFO_NOT_EMPTY BIT(1) +#define RESP_FIFO_RDY BIT(2) +#define HRESP_FROM_NOC_ERR BIT(3) +#define WR_FIFO_EMPTY BIT(9) +#define WR_FIFO_FULL BIT(10) +#define WR_FIFO_OVERRUN BIT(11) +#define TRANSACTION_DONE BIT(16) +#define QSPI_ERR_IRQS (RESP_FIFO_UNDERRUN | HRESP_FROM_NOC_ERR | \ + WR_FIFO_OVERRUN) +#define QSPI_ALL_IRQS (QSPI_ERR_IRQS | RESP_FIFO_RDY | \ + WR_FIFO_EMPTY | WR_FIFO_FULL | \ + TRANSACTION_DONE) + +#define PIO_XFER_CTRL 0x0014 +#define REQUEST_COUNT_MSK 0xffff + +#define PIO_XFER_CFG 0x0018 +#define TRANSFER_DIRECTION BIT(0) +#define MULTI_IO_MODE_MSK 0xe +#define MULTI_IO_MODE_SHFT 1 +#define TRANSFER_FRAGMENT BIT(8) +#define SDR_1BIT 1 +#define SDR_2BIT 2 +#define SDR_4BIT 3 +#define DDR_1BIT 5 +#define DDR_2BIT 6 +#define DDR_4BIT 7 +#define DMA_DESC_SINGLE_SPI 1 +#define DMA_DESC_DUAL_SPI 2 +#define DMA_DESC_QUAD_SPI 3 + +#define PIO_XFER_STATUS 0x001c +#define WR_FIFO_BYTES_MSK 0xffff0000 +#define WR_FIFO_BYTES_SHFT 16 + +#define PIO_DATAOUT_1B 0x0020 +#define PIO_DATAOUT_4B 0x0024 + +#define RD_FIFO_STATUS 0x002c +#define FIFO_EMPTY BIT(11) +#define WR_CNTS_MSK 0x7f0 +#define WR_CNTS_SHFT 4 +#define RDY_64BYTE BIT(3) +#define RDY_32BYTE BIT(2) +#define RDY_16BYTE BIT(1) +#define FIFO_RDY BIT(0) + +#define RD_FIFO_CFG 0x0028 +#define CONTINUOUS_MODE BIT(0) + +#define RD_FIFO_RESET 0x0030 +#define RESET_FIFO BIT(0) + +#define CUR_MEM_ADDR 0x0048 +#define HW_VERSION 0x004c +#define RD_FIFO 0x0050 +#define SAMPLING_CLK_CFG 0x0090 +#define SAMPLING_CLK_STATUS 0x0094 + + +enum qspi_dir { + QSPI_READ, + QSPI_WRITE, +}; + +struct qspi_xfer { + union { + const void *tx_buf; + void *rx_buf; + }; + unsigned int rem_bytes; + unsigned int buswidth; + enum qspi_dir dir; + bool is_last; +}; + +enum qspi_clocks { + QSPI_CLK_CORE, + QSPI_CLK_IFACE, + QSPI_NUM_CLKS +}; + +struct qcom_qspi { + void __iomem *base; + struct device *dev; + struct clk_bulk_data clks[QSPI_NUM_CLKS]; + struct qspi_xfer xfer; + /* Lock to protect data accessed by IRQs */ + spinlock_t lock; +}; + +static u32 qspi_buswidth_to_iomode(struct qcom_qspi *ctrl, + unsigned int buswidth) +{ + switch (buswidth) { + case 1: + return SDR_1BIT << MULTI_IO_MODE_SHFT; + case 2: + return SDR_2BIT << MULTI_IO_MODE_SHFT; + case 4: + return SDR_4BIT << MULTI_IO_MODE_SHFT; + default: + dev_warn_once(ctrl->dev, + "Unexpected bus width: %u\n", buswidth); + return SDR_1BIT << MULTI_IO_MODE_SHFT; + } +} + +static void qcom_qspi_pio_xfer_cfg(struct qcom_qspi *ctrl) +{ + u32 pio_xfer_cfg; + const struct qspi_xfer *xfer; + + xfer = &ctrl->xfer; + pio_xfer_cfg = readl(ctrl->base + PIO_XFER_CFG); + pio_xfer_cfg &= ~TRANSFER_DIRECTION; + pio_xfer_cfg |= xfer->dir; + if (xfer->is_last) + pio_xfer_cfg &= ~TRANSFER_FRAGMENT; + else + pio_xfer_cfg |= TRANSFER_FRAGMENT; + pio_xfer_cfg &= ~MULTI_IO_MODE_MSK; + pio_xfer_cfg |= qspi_buswidth_to_iomode(ctrl, xfer->buswidth); + + writel(pio_xfer_cfg, ctrl->base + PIO_XFER_CFG); +} + +static void qcom_qspi_pio_xfer_ctrl(struct qcom_qspi *ctrl) +{ + u32 pio_xfer_ctrl; + + pio_xfer_ctrl = readl(ctrl->base + PIO_XFER_CTRL); + pio_xfer_ctrl &= ~REQUEST_COUNT_MSK; + pio_xfer_ctrl |= ctrl->xfer.rem_bytes; + writel(pio_xfer_ctrl, ctrl->base + PIO_XFER_CTRL); +} + +static void qcom_qspi_pio_xfer(struct qcom_qspi *ctrl) +{ + u32 ints; + + qcom_qspi_pio_xfer_cfg(ctrl); + + /* Ack any previous interrupts that might be hanging around */ + writel(QSPI_ALL_IRQS, ctrl->base + MSTR_INT_STATUS); + + /* Setup new interrupts */ + if (ctrl->xfer.dir == QSPI_WRITE) + ints = QSPI_ERR_IRQS | WR_FIFO_EMPTY; + else + ints = QSPI_ERR_IRQS | RESP_FIFO_RDY; + writel(ints, ctrl->base + MSTR_INT_EN); + + /* Kick off the transfer */ + qcom_qspi_pio_xfer_ctrl(ctrl); +} + +static void qcom_qspi_handle_err(struct spi_master *master, + struct spi_message *msg) +{ + struct qcom_qspi *ctrl = spi_master_get_devdata(master); + unsigned long flags; + + spin_lock_irqsave(&ctrl->lock, flags); + writel(0, ctrl->base + MSTR_INT_EN); + ctrl->xfer.rem_bytes = 0; + spin_unlock_irqrestore(&ctrl->lock, flags); +} + +static int qcom_qspi_transfer_one(struct spi_master *master, + struct spi_device *slv, + struct spi_transfer *xfer) +{ + struct qcom_qspi *ctrl = spi_master_get_devdata(master); + int ret; + unsigned long speed_hz; + unsigned long flags; + + speed_hz = slv->max_speed_hz; + if (xfer->speed_hz) + speed_hz = xfer->speed_hz; + + /* In regular operation (SBL_EN=1) core must be 4x transfer clock */ + ret = clk_set_rate(ctrl->clks[QSPI_CLK_CORE].clk, speed_hz * 4); + if (ret) { + dev_err(ctrl->dev, "Failed to set core clk %d\n", ret); + return ret; + } + + spin_lock_irqsave(&ctrl->lock, flags); + + /* We are half duplex, so either rx or tx will be set */ + if (xfer->rx_buf) { + ctrl->xfer.dir = QSPI_READ; + ctrl->xfer.buswidth = xfer->rx_nbits; + ctrl->xfer.rx_buf = xfer->rx_buf; + } else { + ctrl->xfer.dir = QSPI_WRITE; + ctrl->xfer.buswidth = xfer->tx_nbits; + ctrl->xfer.tx_buf = xfer->tx_buf; + } + ctrl->xfer.is_last = list_is_last(&xfer->transfer_list, + &master->cur_msg->transfers); + ctrl->xfer.rem_bytes = xfer->len; + qcom_qspi_pio_xfer(ctrl); + + spin_unlock_irqrestore(&ctrl->lock, flags); + + /* We'll call spi_finalize_current_transfer() when done */ + return 1; +} + +static int qcom_qspi_prepare_message(struct spi_master *master, + struct spi_message *message) +{ + u32 mstr_cfg; + struct qcom_qspi *ctrl; + int tx_data_oe_delay = 1; + int tx_data_delay = 1; + unsigned long flags; + + ctrl = spi_master_get_devdata(master); + spin_lock_irqsave(&ctrl->lock, flags); + + mstr_cfg = readl(ctrl->base + MSTR_CONFIG); + mstr_cfg &= ~CHIP_SELECT_NUM; + if (message->spi->chip_select) + mstr_cfg |= CHIP_SELECT_NUM; + + mstr_cfg |= FB_CLK_EN | PIN_WPN | PIN_HOLDN | SBL_EN | FULL_CYCLE_MODE; + mstr_cfg &= ~(SPI_MODE_MSK | TX_DATA_OE_DELAY_MSK | TX_DATA_DELAY_MSK); + mstr_cfg |= message->spi->mode << SPI_MODE_SHFT; + mstr_cfg |= tx_data_oe_delay << TX_DATA_OE_DELAY_SHFT; + mstr_cfg |= tx_data_delay << TX_DATA_DELAY_SHFT; + mstr_cfg &= ~DMA_ENABLE; + + writel(mstr_cfg, ctrl->base + MSTR_CONFIG); + spin_unlock_irqrestore(&ctrl->lock, flags); + + return 0; +} + +static irqreturn_t pio_read(struct qcom_qspi *ctrl) +{ + u32 rd_fifo_status; + u32 rd_fifo; + unsigned int wr_cnts; + unsigned int bytes_to_read; + unsigned int words_to_read; + u32 *word_buf; + u8 *byte_buf; + int i; + + rd_fifo_status = readl(ctrl->base + RD_FIFO_STATUS); + + if (!(rd_fifo_status & FIFO_RDY)) { + dev_dbg(ctrl->dev, "Spurious IRQ %#x\n", rd_fifo_status); + return IRQ_NONE; + } + + wr_cnts = (rd_fifo_status & WR_CNTS_MSK) >> WR_CNTS_SHFT; + wr_cnts = min(wr_cnts, ctrl->xfer.rem_bytes); + + words_to_read = wr_cnts / QSPI_BYTES_PER_WORD; + bytes_to_read = wr_cnts % QSPI_BYTES_PER_WORD; + + if (words_to_read) { + word_buf = ctrl->xfer.rx_buf; + ctrl->xfer.rem_bytes -= words_to_read * QSPI_BYTES_PER_WORD; + ioread32_rep(ctrl->base + RD_FIFO, word_buf, words_to_read); + ctrl->xfer.rx_buf = word_buf + words_to_read; + } + + if (bytes_to_read) { + byte_buf = ctrl->xfer.rx_buf; + rd_fifo = readl(ctrl->base + RD_FIFO); + ctrl->xfer.rem_bytes -= bytes_to_read; + for (i = 0; i < bytes_to_read; i++) + *byte_buf++ = rd_fifo >> (i * BITS_PER_BYTE); + ctrl->xfer.rx_buf = byte_buf; + } + + return IRQ_HANDLED; +} + +static irqreturn_t pio_write(struct qcom_qspi *ctrl) +{ + const void *xfer_buf = ctrl->xfer.tx_buf; + const int *word_buf; + const char *byte_buf; + unsigned int wr_fifo_bytes; + unsigned int wr_fifo_words; + unsigned int wr_size; + unsigned int rem_words; + + wr_fifo_bytes = readl(ctrl->base + PIO_XFER_STATUS); + wr_fifo_bytes >>= WR_FIFO_BYTES_SHFT; + + if (ctrl->xfer.rem_bytes < QSPI_BYTES_PER_WORD) { + /* Process the last 1-3 bytes */ + wr_size = min(wr_fifo_bytes, ctrl->xfer.rem_bytes); + ctrl->xfer.rem_bytes -= wr_size; + + byte_buf = xfer_buf; + while (wr_size--) + writel(*byte_buf++, + ctrl->base + PIO_DATAOUT_1B); + ctrl->xfer.tx_buf = byte_buf; + } else { + /* + * Process all the whole words; to keep things simple we'll + * just wait for the next interrupt to handle the last 1-3 + * bytes if we don't have an even number of words. + */ + rem_words = ctrl->xfer.rem_bytes / QSPI_BYTES_PER_WORD; + wr_fifo_words = wr_fifo_bytes / QSPI_BYTES_PER_WORD; + + wr_size = min(rem_words, wr_fifo_words); + ctrl->xfer.rem_bytes -= wr_size * QSPI_BYTES_PER_WORD; + + word_buf = xfer_buf; + iowrite32_rep(ctrl->base + PIO_DATAOUT_4B, word_buf, wr_size); + ctrl->xfer.tx_buf = word_buf + wr_size; + + } + + return IRQ_HANDLED; +} + +static irqreturn_t qcom_qspi_irq(int irq, void *dev_id) +{ + u32 int_status; + struct qcom_qspi *ctrl = dev_id; + irqreturn_t ret = IRQ_NONE; + unsigned long flags; + + spin_lock_irqsave(&ctrl->lock, flags); + + int_status = readl(ctrl->base + MSTR_INT_STATUS); + writel(int_status, ctrl->base + MSTR_INT_STATUS); + + if (ctrl->xfer.dir == QSPI_WRITE) { + if (int_status & WR_FIFO_EMPTY) + ret = pio_write(ctrl); + } else { + if (int_status & RESP_FIFO_RDY) + ret = pio_read(ctrl); + } + + if (int_status & QSPI_ERR_IRQS) { + if (int_status & RESP_FIFO_UNDERRUN) + dev_err(ctrl->dev, "IRQ error: FIFO underrun\n"); + if (int_status & WR_FIFO_OVERRUN) + dev_err(ctrl->dev, "IRQ error: FIFO overrun\n"); + if (int_status & HRESP_FROM_NOC_ERR) + dev_err(ctrl->dev, "IRQ error: NOC response error\n"); + ret = IRQ_HANDLED; + } + + if (!ctrl->xfer.rem_bytes) { + writel(0, ctrl->base + MSTR_INT_EN); + spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev)); + } + + spin_unlock_irqrestore(&ctrl->lock, flags); + return ret; +} + +static int qcom_qspi_probe(struct platform_device *pdev) +{ + int ret; + struct device *dev; + struct resource *res; + struct spi_master *master; + struct qcom_qspi *ctrl; + + dev = &pdev->dev; + + master = spi_alloc_master(dev, sizeof(*ctrl)); + if (!master) + return -ENOMEM; + + platform_set_drvdata(pdev, master); + + ctrl = spi_master_get_devdata(master); + + spin_lock_init(&ctrl->lock); + ctrl->dev = dev; + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + ctrl->base = devm_ioremap_resource(dev, res); + if (IS_ERR(ctrl->base)) { + ret = PTR_ERR(ctrl->base); + goto exit_probe_master_put; + } + + ctrl->clks[QSPI_CLK_CORE].id = "core"; + ctrl->clks[QSPI_CLK_IFACE].id = "iface"; + ret = devm_clk_bulk_get(dev, QSPI_NUM_CLKS, ctrl->clks); + if (ret) + goto exit_probe_master_put; + + ret = platform_get_irq(pdev, 0); + if (ret < 0) { + dev_err(dev, "Failed to get irq %d\n", ret); + goto exit_probe_master_put; + } + ret = devm_request_irq(dev, ret, qcom_qspi_irq, + IRQF_TRIGGER_HIGH, dev_name(dev), ctrl); + if (ret) { + dev_err(dev, "Failed to request irq %d\n", ret); + goto exit_probe_master_put; + } + + master->max_speed_hz = 300000000; + master->num_chipselect = QSPI_NUM_CS; + master->bus_num = -1; + master->dev.of_node = pdev->dev.of_node; + master->mode_bits = SPI_MODE_0 | + SPI_TX_DUAL | SPI_RX_DUAL | + SPI_TX_QUAD | SPI_RX_QUAD; + master->flags = SPI_MASTER_HALF_DUPLEX; + master->prepare_message = qcom_qspi_prepare_message; + master->transfer_one = qcom_qspi_transfer_one; + master->handle_err = qcom_qspi_handle_err; + master->auto_runtime_pm = true; + + pm_runtime_enable(dev); + + ret = spi_register_master(master); + if (!ret) + return 0; + + pm_runtime_disable(dev); + +exit_probe_master_put: + spi_master_put(master); + + return ret; +} + +static int qcom_qspi_remove(struct platform_device *pdev) +{ + struct spi_master *master = platform_get_drvdata(pdev); + + /* Unregister _before_ disabling pm_runtime() so we stop transfers */ + spi_unregister_master(master); + + pm_runtime_disable(&pdev->dev); + + return 0; +} + +static int __maybe_unused qcom_qspi_runtime_suspend(struct device *dev) +{ + struct spi_master *master = dev_get_drvdata(dev); + struct qcom_qspi *ctrl = spi_master_get_devdata(master); + + clk_bulk_disable_unprepare(QSPI_NUM_CLKS, ctrl->clks); + + return 0; +} + +static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev) +{ + struct spi_master *master = dev_get_drvdata(dev); + struct qcom_qspi *ctrl = spi_master_get_devdata(master); + + return clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks); +} + +static int __maybe_unused qcom_qspi_suspend(struct device *dev) +{ + struct spi_master *master = dev_get_drvdata(dev); + int ret; + + ret = spi_master_suspend(master); + if (ret) + return ret; + + ret = pm_runtime_force_suspend(dev); + if (ret) + spi_master_resume(master); + + return ret; +} + +static int __maybe_unused qcom_qspi_resume(struct device *dev) +{ + struct spi_master *master = dev_get_drvdata(dev); + int ret; + + ret = pm_runtime_force_resume(dev); + if (ret) + return ret; + + ret = spi_master_resume(master); + if (ret) + pm_runtime_force_suspend(dev); + + return ret; +} + +static const struct dev_pm_ops qcom_qspi_dev_pm_ops = { + SET_RUNTIME_PM_OPS(qcom_qspi_runtime_suspend, + qcom_qspi_runtime_resume, NULL) + SET_SYSTEM_SLEEP_PM_OPS(qcom_qspi_suspend, qcom_qspi_resume) +}; + +static const struct of_device_id qcom_qspi_dt_match[] = { + { .compatible = "qcom,qspi-v1", }, + { } +}; +MODULE_DEVICE_TABLE(of, qcom_qspi_dt_match); + +static struct platform_driver qcom_qspi_driver = { + .driver = { + .name = "qcom_qspi", + .pm = &qcom_qspi_dev_pm_ops, + .of_match_table = qcom_qspi_dt_match, + }, + .probe = qcom_qspi_probe, + .remove = qcom_qspi_remove, +}; +module_platform_driver(qcom_qspi_driver); + +MODULE_DESCRIPTION("SPI driver for QSPI cores"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/spi/spi-rb4xx.c b/drivers/spi/spi-rb4xx.c index 3641d0e20135..fbbf9a188247 100644 --- a/drivers/spi/spi-rb4xx.c +++ b/drivers/spi/spi-rb4xx.c @@ -159,7 +159,7 @@ static int rb4xx_spi_probe(struct platform_device *pdev) master->bus_num = 0; master->num_chipselect = 3; master->mode_bits = SPI_TX_DUAL; - master->bits_per_word_mask = BIT(7); + master->bits_per_word_mask = SPI_BPW_MASK(8); master->flags = SPI_MASTER_MUST_TX; master->transfer_one = rb4xx_transfer_one; master->set_cs = rb4xx_set_cs; diff --git a/drivers/spi/spi-rockchip.c b/drivers/spi/spi-rockchip.c index fdcf3076681b..51ef632bca52 100644 --- a/drivers/spi/spi-rockchip.c +++ b/drivers/spi/spi-rockchip.c @@ -164,7 +164,6 @@ enum rockchip_ssi_type { struct rockchip_spi_dma_data { struct dma_chan *ch; - enum dma_transfer_direction direction; dma_addr_t addr; }; @@ -202,12 +201,11 @@ struct rockchip_spi { bool cs_asserted[ROCKCHIP_SPI_MAX_CS_NUM]; - u32 use_dma; + bool use_dma; struct sg_table tx_sg; struct sg_table rx_sg; struct rockchip_spi_dma_data dma_rx; struct rockchip_spi_dma_data dma_tx; - struct dma_slave_caps dma_caps; }; static inline void spi_enable_chip(struct rockchip_spi *rs, int enable) @@ -381,6 +379,8 @@ static int rockchip_spi_pio_transfer(struct rockchip_spi *rs) { int remain = 0; + spi_enable_chip(rs, 1); + do { if (rs->tx) { remain = rs->tx_end - rs->tx; @@ -445,6 +445,9 @@ static int rockchip_spi_prepare_dma(struct rockchip_spi *rs) struct dma_slave_config rxconf, txconf; struct dma_async_tx_descriptor *rxdesc, *txdesc; + memset(&rxconf, 0, sizeof(rxconf)); + memset(&txconf, 0, sizeof(txconf)); + spin_lock_irqsave(&rs->lock, flags); rs->state &= ~RXBUSY; rs->state &= ~TXBUSY; @@ -452,19 +455,16 @@ static int rockchip_spi_prepare_dma(struct rockchip_spi *rs) rxdesc = NULL; if (rs->rx) { - rxconf.direction = rs->dma_rx.direction; + rxconf.direction = DMA_DEV_TO_MEM; rxconf.src_addr = rs->dma_rx.addr; rxconf.src_addr_width = rs->n_bytes; - if (rs->dma_caps.max_burst > 4) - rxconf.src_maxburst = 4; - else - rxconf.src_maxburst = 1; + rxconf.src_maxburst = 1; dmaengine_slave_config(rs->dma_rx.ch, &rxconf); rxdesc = dmaengine_prep_slave_sg( rs->dma_rx.ch, rs->rx_sg.sgl, rs->rx_sg.nents, - rs->dma_rx.direction, DMA_PREP_INTERRUPT); + DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT); if (!rxdesc) return -EINVAL; @@ -474,19 +474,16 @@ static int rockchip_spi_prepare_dma(struct rockchip_spi *rs) txdesc = NULL; if (rs->tx) { - txconf.direction = rs->dma_tx.direction; + txconf.direction = DMA_MEM_TO_DEV; txconf.dst_addr = rs->dma_tx.addr; txconf.dst_addr_width = rs->n_bytes; - if (rs->dma_caps.max_burst > 4) - txconf.dst_maxburst = 4; - else - txconf.dst_maxburst = 1; + txconf.dst_maxburst = rs->fifo_len / 2; dmaengine_slave_config(rs->dma_tx.ch, &txconf); txdesc = dmaengine_prep_slave_sg( rs->dma_tx.ch, rs->tx_sg.sgl, rs->tx_sg.nents, - rs->dma_tx.direction, DMA_PREP_INTERRUPT); + DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT); if (!txdesc) { if (rxdesc) dmaengine_terminate_sync(rs->dma_rx.ch); @@ -506,6 +503,8 @@ static int rockchip_spi_prepare_dma(struct rockchip_spi *rs) dma_async_issue_pending(rs->dma_rx.ch); } + spi_enable_chip(rs, 1); + if (txdesc) { spin_lock_irqsave(&rs->lock, flags); rs->state |= TXBUSY; @@ -514,7 +513,8 @@ static int rockchip_spi_prepare_dma(struct rockchip_spi *rs) dma_async_issue_pending(rs->dma_tx.ch); } - return 0; + /* 1 means the transfer is in progress */ + return 1; } static void rockchip_spi_config(struct rockchip_spi *rs) @@ -578,7 +578,7 @@ static void rockchip_spi_config(struct rockchip_spi *rs) writel_relaxed(rs->fifo_len / 2 - 1, rs->regs + ROCKCHIP_SPI_TXFTLR); writel_relaxed(rs->fifo_len / 2 - 1, rs->regs + ROCKCHIP_SPI_RXFTLR); - writel_relaxed(0, rs->regs + ROCKCHIP_SPI_DMATDLR); + writel_relaxed(rs->fifo_len / 2 - 1, rs->regs + ROCKCHIP_SPI_DMATDLR); writel_relaxed(0, rs->regs + ROCKCHIP_SPI_DMARDLR); writel_relaxed(dmacr, rs->regs + ROCKCHIP_SPI_DMACR); @@ -597,7 +597,6 @@ static int rockchip_spi_transfer_one( struct spi_device *spi, struct spi_transfer *xfer) { - int ret = 0; struct rockchip_spi *rs = spi_master_get_devdata(master); WARN_ON(readl_relaxed(rs->regs + ROCKCHIP_SPI_SSIENR) && @@ -635,30 +634,16 @@ static int rockchip_spi_transfer_one( /* we need prepare dma before spi was enabled */ if (master->can_dma && master->can_dma(master, spi, xfer)) - rs->use_dma = 1; + rs->use_dma = true; else - rs->use_dma = 0; + rs->use_dma = false; rockchip_spi_config(rs); - if (rs->use_dma) { - if (rs->tmode == CR0_XFM_RO) { - /* rx: dma must be prepared first */ - ret = rockchip_spi_prepare_dma(rs); - spi_enable_chip(rs, 1); - } else { - /* tx or tr: spi must be enabled first */ - spi_enable_chip(rs, 1); - ret = rockchip_spi_prepare_dma(rs); - } - /* successful DMA prepare means the transfer is in progress */ - ret = ret ? ret : 1; - } else { - spi_enable_chip(rs, 1); - ret = rockchip_spi_pio_transfer(rs); - } + if (rs->use_dma) + return rockchip_spi_prepare_dma(rs); - return ret; + return rockchip_spi_pio_transfer(rs); } static bool rockchip_spi_can_dma(struct spi_master *master, @@ -780,11 +765,8 @@ static int rockchip_spi_probe(struct platform_device *pdev) } if (rs->dma_tx.ch && rs->dma_rx.ch) { - dma_get_slave_caps(rs->dma_rx.ch, &(rs->dma_caps)); rs->dma_tx.addr = (dma_addr_t)(mem->start + ROCKCHIP_SPI_TXDR); rs->dma_rx.addr = (dma_addr_t)(mem->start + ROCKCHIP_SPI_RXDR); - rs->dma_tx.direction = DMA_MEM_TO_DEV; - rs->dma_rx.direction = DMA_DEV_TO_MEM; master->can_dma = rockchip_spi_can_dma; master->dma_tx = rs->dma_tx.ch; diff --git a/drivers/spi/spi-rspi.c b/drivers/spi/spi-rspi.c index b37de1d991d6..55f8e55327b3 100644 --- a/drivers/spi/spi-rspi.c +++ b/drivers/spi/spi-rspi.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * SH RSPI driver * @@ -6,15 +7,6 @@ * * Based on spi-sh.c: * Copyright (C) 2011 Renesas Solutions Corp. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include <linux/module.h> diff --git a/drivers/spi/spi-sh-hspi.c b/drivers/spi/spi-sh-hspi.c index 20e800e70442..dc0926e43665 100644 --- a/drivers/spi/spi-sh-hspi.c +++ b/drivers/spi/spi-sh-hspi.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * SuperH HSPI bus driver * @@ -7,15 +8,6 @@ * Based on pxa2xx_spi.c: * Copyright (C) 2011 Renesas Solutions Corp. * Copyright (C) 2005 Stephen Street / StreetFire Sound Labs - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include <linux/clk.h> @@ -316,6 +308,6 @@ static struct platform_driver hspi_driver = { module_platform_driver(hspi_driver); MODULE_DESCRIPTION("SuperH HSPI bus driver"); -MODULE_LICENSE("GPL"); +MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>"); MODULE_ALIAS("platform:sh-hspi"); diff --git a/drivers/spi/spi-sh-msiof.c b/drivers/spi/spi-sh-msiof.c index 101cd6aae2ea..adf384323934 100644 --- a/drivers/spi/spi-sh-msiof.c +++ b/drivers/spi/spi-sh-msiof.c @@ -1,14 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * SuperH MSIOF SPI Master Interface * * Copyright (c) 2009 Magnus Damm * Copyright (C) 2014 Renesas Electronics Corporation * Copyright (C) 2014-2017 Glider bvba - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * */ #include <linux/bitmap.h> @@ -1343,8 +1339,8 @@ static int sh_msiof_spi_probe(struct platform_device *pdev) i = platform_get_irq(pdev, 0); if (i < 0) { - dev_err(&pdev->dev, "cannot get platform IRQ\n"); - ret = -ENOENT; + dev_err(&pdev->dev, "cannot get IRQ\n"); + ret = i; goto err1; } diff --git a/drivers/spi/spi-sh.c b/drivers/spi/spi-sh.c index 50e0ea9acf8b..f1ee58208216 100644 --- a/drivers/spi/spi-sh.c +++ b/drivers/spi/spi-sh.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * SH SPI bus driver * @@ -5,15 +6,6 @@ * * Based on pxa2xx_spi.c: * Copyright (C) 2005 Stephen Street / StreetFire Sound Labs - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include <linux/module.h> @@ -522,6 +514,6 @@ static struct platform_driver spi_sh_driver = { module_platform_driver(spi_sh_driver); MODULE_DESCRIPTION("SH SPI bus driver"); -MODULE_LICENSE("GPL"); +MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Yoshihiro Shimoda"); MODULE_ALIAS("platform:sh_spi"); diff --git a/drivers/spi/spi-slave-mt27xx.c b/drivers/spi/spi-slave-mt27xx.c new file mode 100644 index 000000000000..d1075433f6a6 --- /dev/null +++ b/drivers/spi/spi-slave-mt27xx.c @@ -0,0 +1,554 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright (c) 2018 MediaTek Inc. + +#include <linux/clk.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/spi/spi.h> + +#define SPIS_IRQ_EN_REG 0x0 +#define SPIS_IRQ_CLR_REG 0x4 +#define SPIS_IRQ_ST_REG 0x8 +#define SPIS_IRQ_MASK_REG 0xc +#define SPIS_CFG_REG 0x10 +#define SPIS_RX_DATA_REG 0x14 +#define SPIS_TX_DATA_REG 0x18 +#define SPIS_RX_DST_REG 0x1c +#define SPIS_TX_SRC_REG 0x20 +#define SPIS_DMA_CFG_REG 0x30 +#define SPIS_SOFT_RST_REG 0x40 + +/* SPIS_IRQ_EN_REG */ +#define DMA_DONE_EN BIT(7) +#define DATA_DONE_EN BIT(2) +#define RSTA_DONE_EN BIT(1) +#define CMD_INVALID_EN BIT(0) + +/* SPIS_IRQ_ST_REG */ +#define DMA_DONE_ST BIT(7) +#define DATA_DONE_ST BIT(2) +#define RSTA_DONE_ST BIT(1) +#define CMD_INVALID_ST BIT(0) + +/* SPIS_IRQ_MASK_REG */ +#define DMA_DONE_MASK BIT(7) +#define DATA_DONE_MASK BIT(2) +#define RSTA_DONE_MASK BIT(1) +#define CMD_INVALID_MASK BIT(0) + +/* SPIS_CFG_REG */ +#define SPIS_TX_ENDIAN BIT(7) +#define SPIS_RX_ENDIAN BIT(6) +#define SPIS_TXMSBF BIT(5) +#define SPIS_RXMSBF BIT(4) +#define SPIS_CPHA BIT(3) +#define SPIS_CPOL BIT(2) +#define SPIS_TX_EN BIT(1) +#define SPIS_RX_EN BIT(0) + +/* SPIS_DMA_CFG_REG */ +#define TX_DMA_TRIG_EN BIT(31) +#define TX_DMA_EN BIT(30) +#define RX_DMA_EN BIT(29) +#define TX_DMA_LEN 0xfffff + +/* SPIS_SOFT_RST_REG */ +#define SPIS_DMA_ADDR_EN BIT(1) +#define SPIS_SOFT_RST BIT(0) + +#define MTK_SPI_SLAVE_MAX_FIFO_SIZE 512U + +struct mtk_spi_slave { + struct device *dev; + void __iomem *base; + struct clk *spi_clk; + struct completion xfer_done; + struct spi_transfer *cur_transfer; + bool slave_aborted; +}; + +static const struct of_device_id mtk_spi_slave_of_match[] = { + { .compatible = "mediatek,mt2712-spi-slave", }, + {} +}; +MODULE_DEVICE_TABLE(of, mtk_spi_slave_of_match); + +static void mtk_spi_slave_disable_dma(struct mtk_spi_slave *mdata) +{ + u32 reg_val; + + reg_val = readl(mdata->base + SPIS_DMA_CFG_REG); + reg_val &= ~RX_DMA_EN; + reg_val &= ~TX_DMA_EN; + writel(reg_val, mdata->base + SPIS_DMA_CFG_REG); +} + +static void mtk_spi_slave_disable_xfer(struct mtk_spi_slave *mdata) +{ + u32 reg_val; + + reg_val = readl(mdata->base + SPIS_CFG_REG); + reg_val &= ~SPIS_TX_EN; + reg_val &= ~SPIS_RX_EN; + writel(reg_val, mdata->base + SPIS_CFG_REG); +} + +static int mtk_spi_slave_wait_for_completion(struct mtk_spi_slave *mdata) +{ + if (wait_for_completion_interruptible(&mdata->xfer_done) || + mdata->slave_aborted) { + dev_err(mdata->dev, "interrupted\n"); + return -EINTR; + } + + return 0; +} + +static int mtk_spi_slave_prepare_message(struct spi_controller *ctlr, + struct spi_message *msg) +{ + struct mtk_spi_slave *mdata = spi_controller_get_devdata(ctlr); + struct spi_device *spi = msg->spi; + bool cpha, cpol; + u32 reg_val; + + cpha = spi->mode & SPI_CPHA ? 1 : 0; + cpol = spi->mode & SPI_CPOL ? 1 : 0; + + reg_val = readl(mdata->base + SPIS_CFG_REG); + if (cpha) + reg_val |= SPIS_CPHA; + else + reg_val &= ~SPIS_CPHA; + if (cpol) + reg_val |= SPIS_CPOL; + else + reg_val &= ~SPIS_CPOL; + + if (spi->mode & SPI_LSB_FIRST) + reg_val &= ~(SPIS_TXMSBF | SPIS_RXMSBF); + else + reg_val |= SPIS_TXMSBF | SPIS_RXMSBF; + + reg_val &= ~SPIS_TX_ENDIAN; + reg_val &= ~SPIS_RX_ENDIAN; + writel(reg_val, mdata->base + SPIS_CFG_REG); + + return 0; +} + +static int mtk_spi_slave_fifo_transfer(struct spi_controller *ctlr, + struct spi_device *spi, + struct spi_transfer *xfer) +{ + struct mtk_spi_slave *mdata = spi_controller_get_devdata(ctlr); + int reg_val, cnt, remainder, ret; + + writel(SPIS_SOFT_RST, mdata->base + SPIS_SOFT_RST_REG); + + reg_val = readl(mdata->base + SPIS_CFG_REG); + if (xfer->rx_buf) + reg_val |= SPIS_RX_EN; + if (xfer->tx_buf) + reg_val |= SPIS_TX_EN; + writel(reg_val, mdata->base + SPIS_CFG_REG); + + cnt = xfer->len / 4; + if (xfer->tx_buf) + iowrite32_rep(mdata->base + SPIS_TX_DATA_REG, + xfer->tx_buf, cnt); + + remainder = xfer->len % 4; + if (xfer->tx_buf && remainder > 0) { + reg_val = 0; + memcpy(®_val, xfer->tx_buf + cnt * 4, remainder); + writel(reg_val, mdata->base + SPIS_TX_DATA_REG); + } + + ret = mtk_spi_slave_wait_for_completion(mdata); + if (ret) { + mtk_spi_slave_disable_xfer(mdata); + writel(SPIS_SOFT_RST, mdata->base + SPIS_SOFT_RST_REG); + } + + return ret; +} + +static int mtk_spi_slave_dma_transfer(struct spi_controller *ctlr, + struct spi_device *spi, + struct spi_transfer *xfer) +{ + struct mtk_spi_slave *mdata = spi_controller_get_devdata(ctlr); + struct device *dev = mdata->dev; + int reg_val, ret; + + writel(SPIS_SOFT_RST, mdata->base + SPIS_SOFT_RST_REG); + + if (xfer->tx_buf) { + /* tx_buf is a const void* where we need a void * for + * the dma mapping + */ + void *nonconst_tx = (void *)xfer->tx_buf; + + xfer->tx_dma = dma_map_single(dev, nonconst_tx, + xfer->len, DMA_TO_DEVICE); + if (dma_mapping_error(dev, xfer->tx_dma)) { + ret = -ENOMEM; + goto disable_transfer; + } + } + + if (xfer->rx_buf) { + xfer->rx_dma = dma_map_single(dev, xfer->rx_buf, + xfer->len, DMA_FROM_DEVICE); + if (dma_mapping_error(dev, xfer->rx_dma)) { + ret = -ENOMEM; + goto unmap_txdma; + } + } + + writel(xfer->tx_dma, mdata->base + SPIS_TX_SRC_REG); + writel(xfer->rx_dma, mdata->base + SPIS_RX_DST_REG); + + writel(SPIS_DMA_ADDR_EN, mdata->base + SPIS_SOFT_RST_REG); + + /* enable config reg tx rx_enable */ + reg_val = readl(mdata->base + SPIS_CFG_REG); + if (xfer->tx_buf) + reg_val |= SPIS_TX_EN; + if (xfer->rx_buf) + reg_val |= SPIS_RX_EN; + writel(reg_val, mdata->base + SPIS_CFG_REG); + + /* config dma */ + reg_val = 0; + reg_val |= (xfer->len - 1) & TX_DMA_LEN; + writel(reg_val, mdata->base + SPIS_DMA_CFG_REG); + + reg_val = readl(mdata->base + SPIS_DMA_CFG_REG); + if (xfer->tx_buf) + reg_val |= TX_DMA_EN; + if (xfer->rx_buf) + reg_val |= RX_DMA_EN; + reg_val |= TX_DMA_TRIG_EN; + writel(reg_val, mdata->base + SPIS_DMA_CFG_REG); + + ret = mtk_spi_slave_wait_for_completion(mdata); + if (ret) + goto unmap_rxdma; + + return 0; + +unmap_rxdma: + if (xfer->rx_buf) + dma_unmap_single(dev, xfer->rx_dma, + xfer->len, DMA_FROM_DEVICE); + +unmap_txdma: + if (xfer->tx_buf) + dma_unmap_single(dev, xfer->tx_dma, + xfer->len, DMA_TO_DEVICE); + +disable_transfer: + mtk_spi_slave_disable_dma(mdata); + mtk_spi_slave_disable_xfer(mdata); + writel(SPIS_SOFT_RST, mdata->base + SPIS_SOFT_RST_REG); + + return ret; +} + +static int mtk_spi_slave_transfer_one(struct spi_controller *ctlr, + struct spi_device *spi, + struct spi_transfer *xfer) +{ + struct mtk_spi_slave *mdata = spi_controller_get_devdata(ctlr); + + reinit_completion(&mdata->xfer_done); + mdata->slave_aborted = false; + mdata->cur_transfer = xfer; + + if (xfer->len > MTK_SPI_SLAVE_MAX_FIFO_SIZE) + return mtk_spi_slave_dma_transfer(ctlr, spi, xfer); + else + return mtk_spi_slave_fifo_transfer(ctlr, spi, xfer); +} + +static int mtk_spi_slave_setup(struct spi_device *spi) +{ + struct mtk_spi_slave *mdata = spi_controller_get_devdata(spi->master); + u32 reg_val; + + reg_val = DMA_DONE_EN | DATA_DONE_EN | + RSTA_DONE_EN | CMD_INVALID_EN; + writel(reg_val, mdata->base + SPIS_IRQ_EN_REG); + + reg_val = DMA_DONE_MASK | DATA_DONE_MASK | + RSTA_DONE_MASK | CMD_INVALID_MASK; + writel(reg_val, mdata->base + SPIS_IRQ_MASK_REG); + + mtk_spi_slave_disable_dma(mdata); + mtk_spi_slave_disable_xfer(mdata); + + return 0; +} + +static int mtk_slave_abort(struct spi_controller *ctlr) +{ + struct mtk_spi_slave *mdata = spi_controller_get_devdata(ctlr); + + mdata->slave_aborted = true; + complete(&mdata->xfer_done); + + return 0; +} + +static irqreturn_t mtk_spi_slave_interrupt(int irq, void *dev_id) +{ + struct spi_controller *ctlr = dev_id; + struct mtk_spi_slave *mdata = spi_controller_get_devdata(ctlr); + struct spi_transfer *trans = mdata->cur_transfer; + u32 int_status, reg_val, cnt, remainder; + + int_status = readl(mdata->base + SPIS_IRQ_ST_REG); + writel(int_status, mdata->base + SPIS_IRQ_CLR_REG); + + if (!trans) + return IRQ_NONE; + + if ((int_status & DMA_DONE_ST) && + ((int_status & DATA_DONE_ST) || + (int_status & RSTA_DONE_ST))) { + writel(SPIS_SOFT_RST, mdata->base + SPIS_SOFT_RST_REG); + + if (trans->tx_buf) + dma_unmap_single(mdata->dev, trans->tx_dma, + trans->len, DMA_TO_DEVICE); + if (trans->rx_buf) + dma_unmap_single(mdata->dev, trans->rx_dma, + trans->len, DMA_FROM_DEVICE); + + mtk_spi_slave_disable_dma(mdata); + mtk_spi_slave_disable_xfer(mdata); + } + + if ((!(int_status & DMA_DONE_ST)) && + ((int_status & DATA_DONE_ST) || + (int_status & RSTA_DONE_ST))) { + cnt = trans->len / 4; + if (trans->rx_buf) + ioread32_rep(mdata->base + SPIS_RX_DATA_REG, + trans->rx_buf, cnt); + remainder = trans->len % 4; + if (trans->rx_buf && remainder > 0) { + reg_val = readl(mdata->base + SPIS_RX_DATA_REG); + memcpy(trans->rx_buf + (cnt * 4), + ®_val, remainder); + } + + mtk_spi_slave_disable_xfer(mdata); + } + + if (int_status & CMD_INVALID_ST) { + dev_warn(&ctlr->dev, "cmd invalid\n"); + return IRQ_NONE; + } + + mdata->cur_transfer = NULL; + complete(&mdata->xfer_done); + + return IRQ_HANDLED; +} + +static int mtk_spi_slave_probe(struct platform_device *pdev) +{ + struct spi_controller *ctlr; + struct mtk_spi_slave *mdata; + struct resource *res; + int irq, ret; + + ctlr = spi_alloc_slave(&pdev->dev, sizeof(*mdata)); + if (!ctlr) { + dev_err(&pdev->dev, "failed to alloc spi slave\n"); + return -ENOMEM; + } + + ctlr->auto_runtime_pm = true; + ctlr->dev.of_node = pdev->dev.of_node; + ctlr->mode_bits = SPI_CPOL | SPI_CPHA; + ctlr->mode_bits |= SPI_LSB_FIRST; + + ctlr->prepare_message = mtk_spi_slave_prepare_message; + ctlr->transfer_one = mtk_spi_slave_transfer_one; + ctlr->setup = mtk_spi_slave_setup; + ctlr->slave_abort = mtk_slave_abort; + + mdata = spi_controller_get_devdata(ctlr); + + platform_set_drvdata(pdev, ctlr); + + init_completion(&mdata->xfer_done); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + ret = -ENODEV; + dev_err(&pdev->dev, "failed to determine base address\n"); + goto err_put_ctlr; + } + + mdata->dev = &pdev->dev; + + mdata->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(mdata->base)) { + ret = PTR_ERR(mdata->base); + goto err_put_ctlr; + } + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + dev_err(&pdev->dev, "failed to get irq (%d)\n", irq); + ret = irq; + goto err_put_ctlr; + } + + ret = devm_request_irq(&pdev->dev, irq, mtk_spi_slave_interrupt, + IRQF_TRIGGER_NONE, dev_name(&pdev->dev), ctlr); + if (ret) { + dev_err(&pdev->dev, "failed to register irq (%d)\n", ret); + goto err_put_ctlr; + } + + mdata->spi_clk = devm_clk_get(&pdev->dev, "spi"); + if (IS_ERR(mdata->spi_clk)) { + ret = PTR_ERR(mdata->spi_clk); + dev_err(&pdev->dev, "failed to get spi-clk: %d\n", ret); + goto err_put_ctlr; + } + + ret = clk_prepare_enable(mdata->spi_clk); + if (ret < 0) { + dev_err(&pdev->dev, "failed to enable spi_clk (%d)\n", ret); + goto err_put_ctlr; + } + + pm_runtime_enable(&pdev->dev); + + ret = devm_spi_register_controller(&pdev->dev, ctlr); + if (ret) { + dev_err(&pdev->dev, + "failed to register slave controller(%d)\n", ret); + clk_disable_unprepare(mdata->spi_clk); + goto err_disable_runtime_pm; + } + + clk_disable_unprepare(mdata->spi_clk); + + return 0; + +err_disable_runtime_pm: + pm_runtime_disable(&pdev->dev); +err_put_ctlr: + spi_controller_put(ctlr); + + return ret; +} + +static int mtk_spi_slave_remove(struct platform_device *pdev) +{ + pm_runtime_disable(&pdev->dev); + + return 0; +} + +#ifdef CONFIG_PM_SLEEP +static int mtk_spi_slave_suspend(struct device *dev) +{ + struct spi_controller *ctlr = dev_get_drvdata(dev); + struct mtk_spi_slave *mdata = spi_controller_get_devdata(ctlr); + int ret; + + ret = spi_controller_suspend(ctlr); + if (ret) + return ret; + + if (!pm_runtime_suspended(dev)) + clk_disable_unprepare(mdata->spi_clk); + + return ret; +} + +static int mtk_spi_slave_resume(struct device *dev) +{ + struct spi_controller *ctlr = dev_get_drvdata(dev); + struct mtk_spi_slave *mdata = spi_controller_get_devdata(ctlr); + int ret; + + if (!pm_runtime_suspended(dev)) { + ret = clk_prepare_enable(mdata->spi_clk); + if (ret < 0) { + dev_err(dev, "failed to enable spi_clk (%d)\n", ret); + return ret; + } + } + + ret = spi_controller_resume(ctlr); + if (ret < 0) + clk_disable_unprepare(mdata->spi_clk); + + return ret; +} +#endif /* CONFIG_PM_SLEEP */ + +#ifdef CONFIG_PM +static int mtk_spi_slave_runtime_suspend(struct device *dev) +{ + struct spi_controller *ctlr = dev_get_drvdata(dev); + struct mtk_spi_slave *mdata = spi_controller_get_devdata(ctlr); + + clk_disable_unprepare(mdata->spi_clk); + + return 0; +} + +static int mtk_spi_slave_runtime_resume(struct device *dev) +{ + struct spi_controller *ctlr = dev_get_drvdata(dev); + struct mtk_spi_slave *mdata = spi_controller_get_devdata(ctlr); + int ret; + + ret = clk_prepare_enable(mdata->spi_clk); + if (ret < 0) { + dev_err(dev, "failed to enable spi_clk (%d)\n", ret); + return ret; + } + + return 0; +} +#endif /* CONFIG_PM */ + +static const struct dev_pm_ops mtk_spi_slave_pm = { + SET_SYSTEM_SLEEP_PM_OPS(mtk_spi_slave_suspend, mtk_spi_slave_resume) + SET_RUNTIME_PM_OPS(mtk_spi_slave_runtime_suspend, + mtk_spi_slave_runtime_resume, NULL) +}; + +static struct platform_driver mtk_spi_slave_driver = { + .driver = { + .name = "mtk-spi-slave", + .pm = &mtk_spi_slave_pm, + .of_match_table = mtk_spi_slave_of_match, + }, + .probe = mtk_spi_slave_probe, + .remove = mtk_spi_slave_remove, +}; + +module_platform_driver(mtk_spi_slave_driver); + +MODULE_DESCRIPTION("MTK SPI Slave Controller driver"); +MODULE_AUTHOR("Leilk Liu <leilk.liu@mediatek.com>"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:mtk-spi-slave"); diff --git a/drivers/spi/spi-slave-system-control.c b/drivers/spi/spi-slave-system-control.c index c0257e937995..169f3d595f60 100644 --- a/drivers/spi/spi-slave-system-control.c +++ b/drivers/spi/spi-slave-system-control.c @@ -60,6 +60,7 @@ static void spi_slave_system_control_complete(void *arg) case CMD_REBOOT: dev_info(&priv->spi->dev, "Rebooting system...\n"); kernel_restart(NULL); + break; case CMD_POWEROFF: dev_info(&priv->spi->dev, "Powering off system...\n"); diff --git a/drivers/spi/spi-sprd.c b/drivers/spi/spi-sprd.c new file mode 100644 index 000000000000..8daa24eec624 --- /dev/null +++ b/drivers/spi/spi-sprd.c @@ -0,0 +1,745 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2018 Spreadtrum Communications Inc. + +#include <linux/clk.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/spi/spi.h> + +#define SPRD_SPI_TXD 0x0 +#define SPRD_SPI_CLKD 0x4 +#define SPRD_SPI_CTL0 0x8 +#define SPRD_SPI_CTL1 0xc +#define SPRD_SPI_CTL2 0x10 +#define SPRD_SPI_CTL3 0x14 +#define SPRD_SPI_CTL4 0x18 +#define SPRD_SPI_CTL5 0x1c +#define SPRD_SPI_INT_EN 0x20 +#define SPRD_SPI_INT_CLR 0x24 +#define SPRD_SPI_INT_RAW_STS 0x28 +#define SPRD_SPI_INT_MASK_STS 0x2c +#define SPRD_SPI_STS1 0x30 +#define SPRD_SPI_STS2 0x34 +#define SPRD_SPI_DSP_WAIT 0x38 +#define SPRD_SPI_STS3 0x3c +#define SPRD_SPI_CTL6 0x40 +#define SPRD_SPI_STS4 0x44 +#define SPRD_SPI_FIFO_RST 0x48 +#define SPRD_SPI_CTL7 0x4c +#define SPRD_SPI_STS5 0x50 +#define SPRD_SPI_CTL8 0x54 +#define SPRD_SPI_CTL9 0x58 +#define SPRD_SPI_CTL10 0x5c +#define SPRD_SPI_CTL11 0x60 +#define SPRD_SPI_CTL12 0x64 +#define SPRD_SPI_STS6 0x68 +#define SPRD_SPI_STS7 0x6c +#define SPRD_SPI_STS8 0x70 +#define SPRD_SPI_STS9 0x74 + +/* Bits & mask definition for register CTL0 */ +#define SPRD_SPI_SCK_REV BIT(13) +#define SPRD_SPI_NG_TX BIT(1) +#define SPRD_SPI_NG_RX BIT(0) +#define SPRD_SPI_CHNL_LEN_MASK GENMASK(4, 0) +#define SPRD_SPI_CSN_MASK GENMASK(11, 8) +#define SPRD_SPI_CS0_VALID BIT(8) + +/* Bits & mask definition for register SPI_INT_EN */ +#define SPRD_SPI_TX_END_INT_EN BIT(8) +#define SPRD_SPI_RX_END_INT_EN BIT(9) + +/* Bits & mask definition for register SPI_INT_RAW_STS */ +#define SPRD_SPI_TX_END_RAW BIT(8) +#define SPRD_SPI_RX_END_RAW BIT(9) + +/* Bits & mask definition for register SPI_INT_CLR */ +#define SPRD_SPI_TX_END_CLR BIT(8) +#define SPRD_SPI_RX_END_CLR BIT(9) + +/* Bits & mask definition for register INT_MASK_STS */ +#define SPRD_SPI_MASK_RX_END BIT(9) +#define SPRD_SPI_MASK_TX_END BIT(8) + +/* Bits & mask definition for register STS2 */ +#define SPRD_SPI_TX_BUSY BIT(8) + +/* Bits & mask definition for register CTL1 */ +#define SPRD_SPI_RX_MODE BIT(12) +#define SPRD_SPI_TX_MODE BIT(13) +#define SPRD_SPI_RTX_MD_MASK GENMASK(13, 12) + +/* Bits & mask definition for register CTL2 */ +#define SPRD_SPI_DMA_EN BIT(6) + +/* Bits & mask definition for register CTL4 */ +#define SPRD_SPI_START_RX BIT(9) +#define SPRD_SPI_ONLY_RECV_MASK GENMASK(8, 0) + +/* Bits & mask definition for register SPI_INT_CLR */ +#define SPRD_SPI_RX_END_INT_CLR BIT(9) +#define SPRD_SPI_TX_END_INT_CLR BIT(8) + +/* Bits & mask definition for register SPI_INT_RAW */ +#define SPRD_SPI_RX_END_IRQ BIT(9) +#define SPRD_SPI_TX_END_IRQ BIT(8) + +/* Bits & mask definition for register CTL12 */ +#define SPRD_SPI_SW_RX_REQ BIT(0) +#define SPRD_SPI_SW_TX_REQ BIT(1) + +/* Bits & mask definition for register CTL7 */ +#define SPRD_SPI_DATA_LINE2_EN BIT(15) +#define SPRD_SPI_MODE_MASK GENMASK(5, 3) +#define SPRD_SPI_MODE_OFFSET 3 +#define SPRD_SPI_3WIRE_MODE 4 +#define SPRD_SPI_4WIRE_MODE 0 + +/* Bits & mask definition for register CTL8 */ +#define SPRD_SPI_TX_MAX_LEN_MASK GENMASK(19, 0) +#define SPRD_SPI_TX_LEN_H_MASK GENMASK(3, 0) +#define SPRD_SPI_TX_LEN_H_OFFSET 16 + +/* Bits & mask definition for register CTL9 */ +#define SPRD_SPI_TX_LEN_L_MASK GENMASK(15, 0) + +/* Bits & mask definition for register CTL10 */ +#define SPRD_SPI_RX_MAX_LEN_MASK GENMASK(19, 0) +#define SPRD_SPI_RX_LEN_H_MASK GENMASK(3, 0) +#define SPRD_SPI_RX_LEN_H_OFFSET 16 + +/* Bits & mask definition for register CTL11 */ +#define SPRD_SPI_RX_LEN_L_MASK GENMASK(15, 0) + +/* Default & maximum word delay cycles */ +#define SPRD_SPI_MIN_DELAY_CYCLE 14 +#define SPRD_SPI_MAX_DELAY_CYCLE 130 + +#define SPRD_SPI_FIFO_SIZE 32 +#define SPRD_SPI_CHIP_CS_NUM 0x4 +#define SPRD_SPI_CHNL_LEN 2 +#define SPRD_SPI_DEFAULT_SOURCE 26000000 +#define SPRD_SPI_MAX_SPEED_HZ 48000000 +#define SPRD_SPI_AUTOSUSPEND_DELAY 100 + +struct sprd_spi { + void __iomem *base; + struct device *dev; + struct clk *clk; + u32 src_clk; + u32 hw_mode; + u32 trans_len; + u32 trans_mode; + u32 word_delay; + u32 hw_speed_hz; + u32 len; + int status; + const void *tx_buf; + void *rx_buf; + int (*read_bufs)(struct sprd_spi *ss, u32 len); + int (*write_bufs)(struct sprd_spi *ss, u32 len); +}; + +static u32 sprd_spi_transfer_max_timeout(struct sprd_spi *ss, + struct spi_transfer *t) +{ + /* + * The time spent on transmission of the full FIFO data is the maximum + * SPI transmission time. + */ + u32 size = t->bits_per_word * SPRD_SPI_FIFO_SIZE; + u32 bit_time_us = DIV_ROUND_UP(USEC_PER_SEC, ss->hw_speed_hz); + u32 total_time_us = size * bit_time_us; + /* + * There is an interval between data and the data in our SPI hardware, + * so the total transmission time need add the interval time. + */ + u32 interval_cycle = SPRD_SPI_FIFO_SIZE * ss->word_delay; + u32 interval_time_us = DIV_ROUND_UP(interval_cycle * USEC_PER_SEC, + ss->src_clk); + + return total_time_us + interval_time_us; +} + +static int sprd_spi_wait_for_tx_end(struct sprd_spi *ss, struct spi_transfer *t) +{ + u32 val, us; + int ret; + + us = sprd_spi_transfer_max_timeout(ss, t); + ret = readl_relaxed_poll_timeout(ss->base + SPRD_SPI_INT_RAW_STS, val, + val & SPRD_SPI_TX_END_IRQ, 0, us); + if (ret) { + dev_err(ss->dev, "SPI error, spi send timeout!\n"); + return ret; + } + + ret = readl_relaxed_poll_timeout(ss->base + SPRD_SPI_STS2, val, + !(val & SPRD_SPI_TX_BUSY), 0, us); + if (ret) { + dev_err(ss->dev, "SPI error, spi busy timeout!\n"); + return ret; + } + + writel_relaxed(SPRD_SPI_TX_END_INT_CLR, ss->base + SPRD_SPI_INT_CLR); + + return 0; +} + +static int sprd_spi_wait_for_rx_end(struct sprd_spi *ss, struct spi_transfer *t) +{ + u32 val, us; + int ret; + + us = sprd_spi_transfer_max_timeout(ss, t); + ret = readl_relaxed_poll_timeout(ss->base + SPRD_SPI_INT_RAW_STS, val, + val & SPRD_SPI_RX_END_IRQ, 0, us); + if (ret) { + dev_err(ss->dev, "SPI error, spi rx timeout!\n"); + return ret; + } + + writel_relaxed(SPRD_SPI_RX_END_INT_CLR, ss->base + SPRD_SPI_INT_CLR); + + return 0; +} + +static void sprd_spi_tx_req(struct sprd_spi *ss) +{ + writel_relaxed(SPRD_SPI_SW_TX_REQ, ss->base + SPRD_SPI_CTL12); +} + +static void sprd_spi_rx_req(struct sprd_spi *ss) +{ + writel_relaxed(SPRD_SPI_SW_RX_REQ, ss->base + SPRD_SPI_CTL12); +} + +static void sprd_spi_enter_idle(struct sprd_spi *ss) +{ + u32 val = readl_relaxed(ss->base + SPRD_SPI_CTL1); + + val &= ~SPRD_SPI_RTX_MD_MASK; + writel_relaxed(val, ss->base + SPRD_SPI_CTL1); +} + +static void sprd_spi_set_transfer_bits(struct sprd_spi *ss, u32 bits) +{ + u32 val = readl_relaxed(ss->base + SPRD_SPI_CTL0); + + /* Set the valid bits for every transaction */ + val &= ~(SPRD_SPI_CHNL_LEN_MASK << SPRD_SPI_CHNL_LEN); + val |= bits << SPRD_SPI_CHNL_LEN; + writel_relaxed(val, ss->base + SPRD_SPI_CTL0); +} + +static void sprd_spi_set_tx_length(struct sprd_spi *ss, u32 length) +{ + u32 val = readl_relaxed(ss->base + SPRD_SPI_CTL8); + + length &= SPRD_SPI_TX_MAX_LEN_MASK; + val &= ~SPRD_SPI_TX_LEN_H_MASK; + val |= length >> SPRD_SPI_TX_LEN_H_OFFSET; + writel_relaxed(val, ss->base + SPRD_SPI_CTL8); + + val = length & SPRD_SPI_TX_LEN_L_MASK; + writel_relaxed(val, ss->base + SPRD_SPI_CTL9); +} + +static void sprd_spi_set_rx_length(struct sprd_spi *ss, u32 length) +{ + u32 val = readl_relaxed(ss->base + SPRD_SPI_CTL10); + + length &= SPRD_SPI_RX_MAX_LEN_MASK; + val &= ~SPRD_SPI_RX_LEN_H_MASK; + val |= length >> SPRD_SPI_RX_LEN_H_OFFSET; + writel_relaxed(val, ss->base + SPRD_SPI_CTL10); + + val = length & SPRD_SPI_RX_LEN_L_MASK; + writel_relaxed(val, ss->base + SPRD_SPI_CTL11); +} + +static void sprd_spi_chipselect(struct spi_device *sdev, bool cs) +{ + struct spi_controller *sctlr = sdev->controller; + struct sprd_spi *ss = spi_controller_get_devdata(sctlr); + u32 val; + + val = readl_relaxed(ss->base + SPRD_SPI_CTL0); + /* The SPI controller will pull down CS pin if cs is 0 */ + if (!cs) { + val &= ~SPRD_SPI_CS0_VALID; + writel_relaxed(val, ss->base + SPRD_SPI_CTL0); + } else { + val |= SPRD_SPI_CSN_MASK; + writel_relaxed(val, ss->base + SPRD_SPI_CTL0); + } +} + +static int sprd_spi_write_only_receive(struct sprd_spi *ss, u32 len) +{ + u32 val; + + /* Clear the start receive bit and reset receive data number */ + val = readl_relaxed(ss->base + SPRD_SPI_CTL4); + val &= ~(SPRD_SPI_START_RX | SPRD_SPI_ONLY_RECV_MASK); + writel_relaxed(val, ss->base + SPRD_SPI_CTL4); + + /* Set the receive data length */ + val = readl_relaxed(ss->base + SPRD_SPI_CTL4); + val |= len & SPRD_SPI_ONLY_RECV_MASK; + writel_relaxed(val, ss->base + SPRD_SPI_CTL4); + + /* Trigger to receive data */ + val = readl_relaxed(ss->base + SPRD_SPI_CTL4); + val |= SPRD_SPI_START_RX; + writel_relaxed(val, ss->base + SPRD_SPI_CTL4); + + return len; +} + +static int sprd_spi_write_bufs_u8(struct sprd_spi *ss, u32 len) +{ + u8 *tx_p = (u8 *)ss->tx_buf; + int i; + + for (i = 0; i < len; i++) + writeb_relaxed(tx_p[i], ss->base + SPRD_SPI_TXD); + + ss->tx_buf += i; + return i; +} + +static int sprd_spi_write_bufs_u16(struct sprd_spi *ss, u32 len) +{ + u16 *tx_p = (u16 *)ss->tx_buf; + int i; + + for (i = 0; i < len; i++) + writew_relaxed(tx_p[i], ss->base + SPRD_SPI_TXD); + + ss->tx_buf += i << 1; + return i << 1; +} + +static int sprd_spi_write_bufs_u32(struct sprd_spi *ss, u32 len) +{ + u32 *tx_p = (u32 *)ss->tx_buf; + int i; + + for (i = 0; i < len; i++) + writel_relaxed(tx_p[i], ss->base + SPRD_SPI_TXD); + + ss->tx_buf += i << 2; + return i << 2; +} + +static int sprd_spi_read_bufs_u8(struct sprd_spi *ss, u32 len) +{ + u8 *rx_p = (u8 *)ss->rx_buf; + int i; + + for (i = 0; i < len; i++) + rx_p[i] = readb_relaxed(ss->base + SPRD_SPI_TXD); + + ss->rx_buf += i; + return i; +} + +static int sprd_spi_read_bufs_u16(struct sprd_spi *ss, u32 len) +{ + u16 *rx_p = (u16 *)ss->rx_buf; + int i; + + for (i = 0; i < len; i++) + rx_p[i] = readw_relaxed(ss->base + SPRD_SPI_TXD); + + ss->rx_buf += i << 1; + return i << 1; +} + +static int sprd_spi_read_bufs_u32(struct sprd_spi *ss, u32 len) +{ + u32 *rx_p = (u32 *)ss->rx_buf; + int i; + + for (i = 0; i < len; i++) + rx_p[i] = readl_relaxed(ss->base + SPRD_SPI_TXD); + + ss->rx_buf += i << 2; + return i << 2; +} + +static int sprd_spi_txrx_bufs(struct spi_device *sdev, struct spi_transfer *t) +{ + struct sprd_spi *ss = spi_controller_get_devdata(sdev->controller); + u32 trans_len = ss->trans_len, len; + int ret, write_size = 0; + + while (trans_len) { + len = trans_len > SPRD_SPI_FIFO_SIZE ? SPRD_SPI_FIFO_SIZE : + trans_len; + if (ss->trans_mode & SPRD_SPI_TX_MODE) { + sprd_spi_set_tx_length(ss, len); + write_size += ss->write_bufs(ss, len); + + /* + * For our 3 wires mode or dual TX line mode, we need + * to request the controller to transfer. + */ + if (ss->hw_mode & SPI_3WIRE || ss->hw_mode & SPI_TX_DUAL) + sprd_spi_tx_req(ss); + + ret = sprd_spi_wait_for_tx_end(ss, t); + } else { + sprd_spi_set_rx_length(ss, len); + + /* + * For our 3 wires mode or dual TX line mode, we need + * to request the controller to read. + */ + if (ss->hw_mode & SPI_3WIRE || ss->hw_mode & SPI_TX_DUAL) + sprd_spi_rx_req(ss); + else + write_size += ss->write_bufs(ss, len); + + ret = sprd_spi_wait_for_rx_end(ss, t); + } + + if (ret) + goto complete; + + if (ss->trans_mode & SPRD_SPI_RX_MODE) + ss->read_bufs(ss, len); + + trans_len -= len; + } + + ret = write_size; + +complete: + sprd_spi_enter_idle(ss); + + return ret; +} + +static void sprd_spi_set_speed(struct sprd_spi *ss, u32 speed_hz) +{ + /* + * From SPI datasheet, the prescale calculation formula: + * prescale = SPI source clock / (2 * SPI_freq) - 1; + */ + u32 clk_div = DIV_ROUND_UP(ss->src_clk, speed_hz << 1) - 1; + + /* Save the real hardware speed */ + ss->hw_speed_hz = (ss->src_clk >> 1) / (clk_div + 1); + writel_relaxed(clk_div, ss->base + SPRD_SPI_CLKD); +} + +static void sprd_spi_init_hw(struct sprd_spi *ss, struct spi_transfer *t) +{ + u16 word_delay, interval; + u32 val; + + val = readl_relaxed(ss->base + SPRD_SPI_CTL7); + val &= ~(SPRD_SPI_SCK_REV | SPRD_SPI_NG_TX | SPRD_SPI_NG_RX); + /* Set default chip selection, clock phase and clock polarity */ + val |= ss->hw_mode & SPI_CPHA ? SPRD_SPI_NG_RX : SPRD_SPI_NG_TX; + val |= ss->hw_mode & SPI_CPOL ? SPRD_SPI_SCK_REV : 0; + writel_relaxed(val, ss->base + SPRD_SPI_CTL0); + + /* + * Set the intervals of two SPI frames, and the inteval calculation + * formula as below per datasheet: + * interval time (source clock cycles) = interval * 4 + 10. + */ + word_delay = clamp_t(u16, t->word_delay, SPRD_SPI_MIN_DELAY_CYCLE, + SPRD_SPI_MAX_DELAY_CYCLE); + interval = DIV_ROUND_UP(word_delay - 10, 4); + ss->word_delay = interval * 4 + 10; + writel_relaxed(interval, ss->base + SPRD_SPI_CTL5); + + /* Reset SPI fifo */ + writel_relaxed(1, ss->base + SPRD_SPI_FIFO_RST); + writel_relaxed(0, ss->base + SPRD_SPI_FIFO_RST); + + /* Set SPI work mode */ + val = readl_relaxed(ss->base + SPRD_SPI_CTL7); + val &= ~SPRD_SPI_MODE_MASK; + + if (ss->hw_mode & SPI_3WIRE) + val |= SPRD_SPI_3WIRE_MODE << SPRD_SPI_MODE_OFFSET; + else + val |= SPRD_SPI_4WIRE_MODE << SPRD_SPI_MODE_OFFSET; + + if (ss->hw_mode & SPI_TX_DUAL) + val |= SPRD_SPI_DATA_LINE2_EN; + else + val &= ~SPRD_SPI_DATA_LINE2_EN; + + writel_relaxed(val, ss->base + SPRD_SPI_CTL7); +} + +static int sprd_spi_setup_transfer(struct spi_device *sdev, + struct spi_transfer *t) +{ + struct sprd_spi *ss = spi_controller_get_devdata(sdev->controller); + u8 bits_per_word = t->bits_per_word; + u32 val, mode = 0; + + ss->len = t->len; + ss->tx_buf = t->tx_buf; + ss->rx_buf = t->rx_buf; + + ss->hw_mode = sdev->mode; + sprd_spi_init_hw(ss, t); + + /* Set tansfer speed and valid bits */ + sprd_spi_set_speed(ss, t->speed_hz); + sprd_spi_set_transfer_bits(ss, bits_per_word); + + if (bits_per_word > 16) + bits_per_word = round_up(bits_per_word, 16); + else + bits_per_word = round_up(bits_per_word, 8); + + switch (bits_per_word) { + case 8: + ss->trans_len = t->len; + ss->read_bufs = sprd_spi_read_bufs_u8; + ss->write_bufs = sprd_spi_write_bufs_u8; + break; + case 16: + ss->trans_len = t->len >> 1; + ss->read_bufs = sprd_spi_read_bufs_u16; + ss->write_bufs = sprd_spi_write_bufs_u16; + break; + case 32: + ss->trans_len = t->len >> 2; + ss->read_bufs = sprd_spi_read_bufs_u32; + ss->write_bufs = sprd_spi_write_bufs_u32; + break; + default: + return -EINVAL; + } + + /* Set transfer read or write mode */ + val = readl_relaxed(ss->base + SPRD_SPI_CTL1); + val &= ~SPRD_SPI_RTX_MD_MASK; + if (t->tx_buf) + mode |= SPRD_SPI_TX_MODE; + if (t->rx_buf) + mode |= SPRD_SPI_RX_MODE; + + writel_relaxed(val | mode, ss->base + SPRD_SPI_CTL1); + + ss->trans_mode = mode; + + /* + * If in only receive mode, we need to trigger the SPI controller to + * receive data automatically. + */ + if (ss->trans_mode == SPRD_SPI_RX_MODE) + ss->write_bufs = sprd_spi_write_only_receive; + + return 0; +} + +static int sprd_spi_transfer_one(struct spi_controller *sctlr, + struct spi_device *sdev, + struct spi_transfer *t) +{ + int ret; + + ret = sprd_spi_setup_transfer(sdev, t); + if (ret) + goto setup_err; + + ret = sprd_spi_txrx_bufs(sdev, t); + if (ret == t->len) + ret = 0; + else if (ret >= 0) + ret = -EREMOTEIO; + +setup_err: + spi_finalize_current_transfer(sctlr); + + return ret; +} + +static int sprd_spi_clk_init(struct platform_device *pdev, struct sprd_spi *ss) +{ + struct clk *clk_spi, *clk_parent; + + clk_spi = devm_clk_get(&pdev->dev, "spi"); + if (IS_ERR(clk_spi)) { + dev_warn(&pdev->dev, "can't get the spi clock\n"); + clk_spi = NULL; + } + + clk_parent = devm_clk_get(&pdev->dev, "source"); + if (IS_ERR(clk_parent)) { + dev_warn(&pdev->dev, "can't get the source clock\n"); + clk_parent = NULL; + } + + ss->clk = devm_clk_get(&pdev->dev, "enable"); + if (IS_ERR(ss->clk)) { + dev_err(&pdev->dev, "can't get the enable clock\n"); + return PTR_ERR(ss->clk); + } + + if (!clk_set_parent(clk_spi, clk_parent)) + ss->src_clk = clk_get_rate(clk_spi); + else + ss->src_clk = SPRD_SPI_DEFAULT_SOURCE; + + return 0; +} + +static int sprd_spi_probe(struct platform_device *pdev) +{ + struct spi_controller *sctlr; + struct resource *res; + struct sprd_spi *ss; + int ret; + + pdev->id = of_alias_get_id(pdev->dev.of_node, "spi"); + sctlr = spi_alloc_master(&pdev->dev, sizeof(*ss)); + if (!sctlr) + return -ENOMEM; + + ss = spi_controller_get_devdata(sctlr); + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + ss->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(ss->base)) { + ret = PTR_ERR(ss->base); + goto free_controller; + } + + ss->dev = &pdev->dev; + sctlr->dev.of_node = pdev->dev.of_node; + sctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_3WIRE | SPI_TX_DUAL; + sctlr->bus_num = pdev->id; + sctlr->set_cs = sprd_spi_chipselect; + sctlr->transfer_one = sprd_spi_transfer_one; + sctlr->auto_runtime_pm = true; + sctlr->max_speed_hz = min_t(u32, ss->src_clk >> 1, + SPRD_SPI_MAX_SPEED_HZ); + + platform_set_drvdata(pdev, sctlr); + ret = sprd_spi_clk_init(pdev, ss); + if (ret) + goto free_controller; + + ret = clk_prepare_enable(ss->clk); + if (ret) + goto free_controller; + + ret = pm_runtime_set_active(&pdev->dev); + if (ret < 0) + goto disable_clk; + + pm_runtime_set_autosuspend_delay(&pdev->dev, + SPRD_SPI_AUTOSUSPEND_DELAY); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_enable(&pdev->dev); + ret = pm_runtime_get_sync(&pdev->dev); + if (ret < 0) { + dev_err(&pdev->dev, "failed to resume SPI controller\n"); + goto err_rpm_put; + } + + ret = devm_spi_register_controller(&pdev->dev, sctlr); + if (ret) + goto err_rpm_put; + + pm_runtime_mark_last_busy(&pdev->dev); + pm_runtime_put_autosuspend(&pdev->dev); + + return 0; + +err_rpm_put: + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_disable(&pdev->dev); +disable_clk: + clk_disable_unprepare(ss->clk); +free_controller: + spi_controller_put(sctlr); + + return ret; +} + +static int sprd_spi_remove(struct platform_device *pdev) +{ + struct spi_controller *sctlr = platform_get_drvdata(pdev); + struct sprd_spi *ss = spi_controller_get_devdata(sctlr); + int ret; + + ret = pm_runtime_get_sync(ss->dev); + if (ret < 0) { + dev_err(ss->dev, "failed to resume SPI controller\n"); + return ret; + } + + clk_disable_unprepare(ss->clk); + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_disable(&pdev->dev); + + return 0; +} + +static int __maybe_unused sprd_spi_runtime_suspend(struct device *dev) +{ + struct spi_controller *sctlr = dev_get_drvdata(dev); + struct sprd_spi *ss = spi_controller_get_devdata(sctlr); + + clk_disable_unprepare(ss->clk); + + return 0; +} + +static int __maybe_unused sprd_spi_runtime_resume(struct device *dev) +{ + struct spi_controller *sctlr = dev_get_drvdata(dev); + struct sprd_spi *ss = spi_controller_get_devdata(sctlr); + int ret; + + ret = clk_prepare_enable(ss->clk); + if (ret) + return ret; + + return 0; +} + +static const struct dev_pm_ops sprd_spi_pm_ops = { + SET_RUNTIME_PM_OPS(sprd_spi_runtime_suspend, + sprd_spi_runtime_resume, NULL) +}; + +static const struct of_device_id sprd_spi_of_match[] = { + { .compatible = "sprd,sc9860-spi", }, + { /* sentinel */ } +}; + +static struct platform_driver sprd_spi_driver = { + .driver = { + .name = "sprd-spi", + .of_match_table = sprd_spi_of_match, + .pm = &sprd_spi_pm_ops, + }, + .probe = sprd_spi_probe, + .remove = sprd_spi_remove, +}; + +module_platform_driver(sprd_spi_driver); + +MODULE_DESCRIPTION("Spreadtrum SPI Controller driver"); +MODULE_AUTHOR("Lanqing Liu <lanqing.liu@spreadtrum.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/spi/spi-stm32-qspi.c b/drivers/spi/spi-stm32-qspi.c new file mode 100644 index 000000000000..3b2a9a6b990d --- /dev/null +++ b/drivers/spi/spi-stm32-qspi.c @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) STMicroelectronics 2018 - All Rights Reserved + * Author: Ludovic Barre <ludovic.barre@st.com> for STMicroelectronics. + */ +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/errno.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/reset.h> +#include <linux/sizes.h> +#include <linux/spi/spi-mem.h> + +#define QSPI_CR 0x00 +#define CR_EN BIT(0) +#define CR_ABORT BIT(1) +#define CR_DMAEN BIT(2) +#define CR_TCEN BIT(3) +#define CR_SSHIFT BIT(4) +#define CR_DFM BIT(6) +#define CR_FSEL BIT(7) +#define CR_FTHRES_MASK GENMASK(12, 8) +#define CR_TEIE BIT(16) +#define CR_TCIE BIT(17) +#define CR_FTIE BIT(18) +#define CR_SMIE BIT(19) +#define CR_TOIE BIT(20) +#define CR_PRESC_MASK GENMASK(31, 24) + +#define QSPI_DCR 0x04 +#define DCR_FSIZE_MASK GENMASK(20, 16) + +#define QSPI_SR 0x08 +#define SR_TEF BIT(0) +#define SR_TCF BIT(1) +#define SR_FTF BIT(2) +#define SR_SMF BIT(3) +#define SR_TOF BIT(4) +#define SR_BUSY BIT(5) +#define SR_FLEVEL_MASK GENMASK(13, 8) + +#define QSPI_FCR 0x0c +#define FCR_CTEF BIT(0) +#define FCR_CTCF BIT(1) + +#define QSPI_DLR 0x10 + +#define QSPI_CCR 0x14 +#define CCR_INST_MASK GENMASK(7, 0) +#define CCR_IMODE_MASK GENMASK(9, 8) +#define CCR_ADMODE_MASK GENMASK(11, 10) +#define CCR_ADSIZE_MASK GENMASK(13, 12) +#define CCR_DCYC_MASK GENMASK(22, 18) +#define CCR_DMODE_MASK GENMASK(25, 24) +#define CCR_FMODE_MASK GENMASK(27, 26) +#define CCR_FMODE_INDW (0U << 26) +#define CCR_FMODE_INDR (1U << 26) +#define CCR_FMODE_APM (2U << 26) +#define CCR_FMODE_MM (3U << 26) +#define CCR_BUSWIDTH_0 0x0 +#define CCR_BUSWIDTH_1 0x1 +#define CCR_BUSWIDTH_2 0x2 +#define CCR_BUSWIDTH_4 0x3 + +#define QSPI_AR 0x18 +#define QSPI_ABR 0x1c +#define QSPI_DR 0x20 +#define QSPI_PSMKR 0x24 +#define QSPI_PSMAR 0x28 +#define QSPI_PIR 0x2c +#define QSPI_LPTR 0x30 +#define LPTR_DFT_TIMEOUT 0x10 + +#define STM32_QSPI_MAX_MMAP_SZ SZ_256M +#define STM32_QSPI_MAX_NORCHIP 2 + +#define STM32_FIFO_TIMEOUT_US 30000 +#define STM32_BUSY_TIMEOUT_US 100000 +#define STM32_ABT_TIMEOUT_US 100000 + +struct stm32_qspi_flash { + struct stm32_qspi *qspi; + u32 cs; + u32 presc; +}; + +struct stm32_qspi { + struct device *dev; + void __iomem *io_base; + void __iomem *mm_base; + resource_size_t mm_size; + struct clk *clk; + u32 clk_rate; + struct stm32_qspi_flash flash[STM32_QSPI_MAX_NORCHIP]; + struct completion data_completion; + u32 fmode; + + /* + * to protect device configuration, could be different between + * 2 flash access (bk1, bk2) + */ + struct mutex lock; +}; + +static irqreturn_t stm32_qspi_irq(int irq, void *dev_id) +{ + struct stm32_qspi *qspi = (struct stm32_qspi *)dev_id; + u32 cr, sr; + + sr = readl_relaxed(qspi->io_base + QSPI_SR); + + if (sr & (SR_TEF | SR_TCF)) { + /* disable irq */ + cr = readl_relaxed(qspi->io_base + QSPI_CR); + cr &= ~CR_TCIE & ~CR_TEIE; + writel_relaxed(cr, qspi->io_base + QSPI_CR); + complete(&qspi->data_completion); + } + + return IRQ_HANDLED; +} + +static void stm32_qspi_read_fifo(u8 *val, void __iomem *addr) +{ + *val = readb_relaxed(addr); +} + +static void stm32_qspi_write_fifo(u8 *val, void __iomem *addr) +{ + writeb_relaxed(*val, addr); +} + +static int stm32_qspi_tx_poll(struct stm32_qspi *qspi, + const struct spi_mem_op *op) +{ + void (*tx_fifo)(u8 *val, void __iomem *addr); + u32 len = op->data.nbytes, sr; + u8 *buf; + int ret; + + if (op->data.dir == SPI_MEM_DATA_IN) { + tx_fifo = stm32_qspi_read_fifo; + buf = op->data.buf.in; + + } else { + tx_fifo = stm32_qspi_write_fifo; + buf = (u8 *)op->data.buf.out; + } + + while (len--) { + ret = readl_relaxed_poll_timeout_atomic(qspi->io_base + QSPI_SR, + sr, (sr & SR_FTF), 1, + STM32_FIFO_TIMEOUT_US); + if (ret) { + dev_err(qspi->dev, "fifo timeout (len:%d stat:%#x)\n", + len, sr); + return ret; + } + tx_fifo(buf++, qspi->io_base + QSPI_DR); + } + + return 0; +} + +static int stm32_qspi_tx_mm(struct stm32_qspi *qspi, + const struct spi_mem_op *op) +{ + memcpy_fromio(op->data.buf.in, qspi->mm_base + op->addr.val, + op->data.nbytes); + return 0; +} + +static int stm32_qspi_tx(struct stm32_qspi *qspi, const struct spi_mem_op *op) +{ + if (!op->data.nbytes) + return 0; + + if (qspi->fmode == CCR_FMODE_MM) + return stm32_qspi_tx_mm(qspi, op); + + return stm32_qspi_tx_poll(qspi, op); +} + +static int stm32_qspi_wait_nobusy(struct stm32_qspi *qspi) +{ + u32 sr; + + return readl_relaxed_poll_timeout_atomic(qspi->io_base + QSPI_SR, sr, + !(sr & SR_BUSY), 1, + STM32_BUSY_TIMEOUT_US); +} + +static int stm32_qspi_wait_cmd(struct stm32_qspi *qspi, + const struct spi_mem_op *op) +{ + u32 cr, sr; + int err = 0; + + if (!op->data.nbytes) + return stm32_qspi_wait_nobusy(qspi); + + if (readl_relaxed(qspi->io_base + QSPI_SR) & SR_TCF) + goto out; + + reinit_completion(&qspi->data_completion); + cr = readl_relaxed(qspi->io_base + QSPI_CR); + writel_relaxed(cr | CR_TCIE | CR_TEIE, qspi->io_base + QSPI_CR); + + if (!wait_for_completion_interruptible_timeout(&qspi->data_completion, + msecs_to_jiffies(1000))) { + err = -ETIMEDOUT; + } else { + sr = readl_relaxed(qspi->io_base + QSPI_SR); + if (sr & SR_TEF) + err = -EIO; + } + +out: + /* clear flags */ + writel_relaxed(FCR_CTCF | FCR_CTEF, qspi->io_base + QSPI_FCR); + + return err; +} + +static int stm32_qspi_get_mode(struct stm32_qspi *qspi, u8 buswidth) +{ + if (buswidth == 4) + return CCR_BUSWIDTH_4; + + return buswidth; +} + +static int stm32_qspi_send(struct spi_mem *mem, const struct spi_mem_op *op) +{ + struct stm32_qspi *qspi = spi_controller_get_devdata(mem->spi->master); + struct stm32_qspi_flash *flash = &qspi->flash[mem->spi->chip_select]; + u32 ccr, cr, addr_max; + int timeout, err = 0; + + dev_dbg(qspi->dev, "cmd:%#x mode:%d.%d.%d.%d addr:%#llx len:%#x\n", + op->cmd.opcode, op->cmd.buswidth, op->addr.buswidth, + op->dummy.buswidth, op->data.buswidth, + op->addr.val, op->data.nbytes); + + err = stm32_qspi_wait_nobusy(qspi); + if (err) + goto abort; + + addr_max = op->addr.val + op->data.nbytes + 1; + + if (op->data.dir == SPI_MEM_DATA_IN) { + if (addr_max < qspi->mm_size && + op->addr.buswidth) + qspi->fmode = CCR_FMODE_MM; + else + qspi->fmode = CCR_FMODE_INDR; + } else { + qspi->fmode = CCR_FMODE_INDW; + } + + cr = readl_relaxed(qspi->io_base + QSPI_CR); + cr &= ~CR_PRESC_MASK & ~CR_FSEL; + cr |= FIELD_PREP(CR_PRESC_MASK, flash->presc); + cr |= FIELD_PREP(CR_FSEL, flash->cs); + writel_relaxed(cr, qspi->io_base + QSPI_CR); + + if (op->data.nbytes) + writel_relaxed(op->data.nbytes - 1, + qspi->io_base + QSPI_DLR); + else + qspi->fmode = CCR_FMODE_INDW; + + ccr = qspi->fmode; + ccr |= FIELD_PREP(CCR_INST_MASK, op->cmd.opcode); + ccr |= FIELD_PREP(CCR_IMODE_MASK, + stm32_qspi_get_mode(qspi, op->cmd.buswidth)); + + if (op->addr.nbytes) { + ccr |= FIELD_PREP(CCR_ADMODE_MASK, + stm32_qspi_get_mode(qspi, op->addr.buswidth)); + ccr |= FIELD_PREP(CCR_ADSIZE_MASK, op->addr.nbytes - 1); + } + + if (op->dummy.buswidth && op->dummy.nbytes) + ccr |= FIELD_PREP(CCR_DCYC_MASK, + op->dummy.nbytes * 8 / op->dummy.buswidth); + + if (op->data.nbytes) { + ccr |= FIELD_PREP(CCR_DMODE_MASK, + stm32_qspi_get_mode(qspi, op->data.buswidth)); + } + + writel_relaxed(ccr, qspi->io_base + QSPI_CCR); + + if (op->addr.nbytes && qspi->fmode != CCR_FMODE_MM) + writel_relaxed(op->addr.val, qspi->io_base + QSPI_AR); + + err = stm32_qspi_tx(qspi, op); + + /* + * Abort in: + * -error case + * -read memory map: prefetching must be stopped if we read the last + * byte of device (device size - fifo size). like device size is not + * knows, the prefetching is always stop. + */ + if (err || qspi->fmode == CCR_FMODE_MM) + goto abort; + + /* wait end of tx in indirect mode */ + err = stm32_qspi_wait_cmd(qspi, op); + if (err) + goto abort; + + return 0; + +abort: + cr = readl_relaxed(qspi->io_base + QSPI_CR) | CR_ABORT; + writel_relaxed(cr, qspi->io_base + QSPI_CR); + + /* wait clear of abort bit by hw */ + timeout = readl_relaxed_poll_timeout_atomic(qspi->io_base + QSPI_CR, + cr, !(cr & CR_ABORT), 1, + STM32_ABT_TIMEOUT_US); + + writel_relaxed(FCR_CTCF, qspi->io_base + QSPI_FCR); + + if (err || timeout) + dev_err(qspi->dev, "%s err:%d abort timeout:%d\n", + __func__, err, timeout); + + return err; +} + +static int stm32_qspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) +{ + struct stm32_qspi *qspi = spi_controller_get_devdata(mem->spi->master); + int ret; + + mutex_lock(&qspi->lock); + ret = stm32_qspi_send(mem, op); + mutex_unlock(&qspi->lock); + + return ret; +} + +static int stm32_qspi_setup(struct spi_device *spi) +{ + struct spi_controller *ctrl = spi->master; + struct stm32_qspi *qspi = spi_controller_get_devdata(ctrl); + struct stm32_qspi_flash *flash; + u32 cr, presc; + + if (ctrl->busy) + return -EBUSY; + + if (!spi->max_speed_hz) + return -EINVAL; + + presc = DIV_ROUND_UP(qspi->clk_rate, spi->max_speed_hz) - 1; + + flash = &qspi->flash[spi->chip_select]; + flash->qspi = qspi; + flash->cs = spi->chip_select; + flash->presc = presc; + + mutex_lock(&qspi->lock); + writel_relaxed(LPTR_DFT_TIMEOUT, qspi->io_base + QSPI_LPTR); + cr = FIELD_PREP(CR_FTHRES_MASK, 3) | CR_TCEN | CR_SSHIFT | CR_EN; + writel_relaxed(cr, qspi->io_base + QSPI_CR); + + /* set dcr fsize to max address */ + writel_relaxed(DCR_FSIZE_MASK, qspi->io_base + QSPI_DCR); + mutex_unlock(&qspi->lock); + + return 0; +} + +/* + * no special host constraint, so use default spi_mem_default_supports_op + * to check supported mode. + */ +static const struct spi_controller_mem_ops stm32_qspi_mem_ops = { + .exec_op = stm32_qspi_exec_op, +}; + +static void stm32_qspi_release(struct stm32_qspi *qspi) +{ + /* disable qspi */ + writel_relaxed(0, qspi->io_base + QSPI_CR); + mutex_destroy(&qspi->lock); + clk_disable_unprepare(qspi->clk); +} + +static int stm32_qspi_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct spi_controller *ctrl; + struct reset_control *rstc; + struct stm32_qspi *qspi; + struct resource *res; + int ret, irq; + + ctrl = spi_alloc_master(dev, sizeof(*qspi)); + if (!ctrl) + return -ENOMEM; + + qspi = spi_controller_get_devdata(ctrl); + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "qspi"); + qspi->io_base = devm_ioremap_resource(dev, res); + if (IS_ERR(qspi->io_base)) + return PTR_ERR(qspi->io_base); + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "qspi_mm"); + qspi->mm_base = devm_ioremap_resource(dev, res); + if (IS_ERR(qspi->mm_base)) + return PTR_ERR(qspi->mm_base); + + qspi->mm_size = resource_size(res); + if (qspi->mm_size > STM32_QSPI_MAX_MMAP_SZ) + return -EINVAL; + + irq = platform_get_irq(pdev, 0); + ret = devm_request_irq(dev, irq, stm32_qspi_irq, 0, + dev_name(dev), qspi); + if (ret) { + dev_err(dev, "failed to request irq\n"); + return ret; + } + + init_completion(&qspi->data_completion); + + qspi->clk = devm_clk_get(dev, NULL); + if (IS_ERR(qspi->clk)) + return PTR_ERR(qspi->clk); + + qspi->clk_rate = clk_get_rate(qspi->clk); + if (!qspi->clk_rate) + return -EINVAL; + + ret = clk_prepare_enable(qspi->clk); + if (ret) { + dev_err(dev, "can not enable the clock\n"); + return ret; + } + + rstc = devm_reset_control_get_exclusive(dev, NULL); + if (!IS_ERR(rstc)) { + reset_control_assert(rstc); + udelay(2); + reset_control_deassert(rstc); + } + + qspi->dev = dev; + platform_set_drvdata(pdev, qspi); + mutex_init(&qspi->lock); + + ctrl->mode_bits = SPI_RX_DUAL | SPI_RX_QUAD + | SPI_TX_DUAL | SPI_TX_QUAD; + ctrl->setup = stm32_qspi_setup; + ctrl->bus_num = -1; + ctrl->mem_ops = &stm32_qspi_mem_ops; + ctrl->num_chipselect = STM32_QSPI_MAX_NORCHIP; + ctrl->dev.of_node = dev->of_node; + + ret = devm_spi_register_master(dev, ctrl); + if (ret) + goto err_spi_register; + + return 0; + +err_spi_register: + stm32_qspi_release(qspi); + + return ret; +} + +static int stm32_qspi_remove(struct platform_device *pdev) +{ + struct stm32_qspi *qspi = platform_get_drvdata(pdev); + + stm32_qspi_release(qspi); + return 0; +} + +static const struct of_device_id stm32_qspi_match[] = { + {.compatible = "st,stm32f469-qspi"}, + {} +}; +MODULE_DEVICE_TABLE(of, stm32_qspi_match); + +static struct platform_driver stm32_qspi_driver = { + .probe = stm32_qspi_probe, + .remove = stm32_qspi_remove, + .driver = { + .name = "stm32-qspi", + .of_match_table = stm32_qspi_match, + }, +}; +module_platform_driver(stm32_qspi_driver); + +MODULE_AUTHOR("Ludovic Barre <ludovic.barre@st.com>"); +MODULE_DESCRIPTION("STMicroelectronics STM32 quad spi driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 9da0bc5a036c..6ca59406b0b7 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -1,18 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * SPI init/core code * * Copyright (C) 2005 David Brownell * Copyright (C) 2008 Secret Lab Technologies Ltd. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include <linux/kernel.h> @@ -60,6 +51,7 @@ static void spidev_release(struct device *dev) spi->controller->cleanup(spi); spi_controller_put(spi->controller); + kfree(spi->driver_override); kfree(spi); } @@ -77,6 +69,51 @@ modalias_show(struct device *dev, struct device_attribute *a, char *buf) } static DEVICE_ATTR_RO(modalias); +static ssize_t driver_override_store(struct device *dev, + struct device_attribute *a, + const char *buf, size_t count) +{ + struct spi_device *spi = to_spi_device(dev); + const char *end = memchr(buf, '\n', count); + const size_t len = end ? end - buf : count; + const char *driver_override, *old; + + /* We need to keep extra room for a newline when displaying value */ + if (len >= (PAGE_SIZE - 1)) + return -EINVAL; + + driver_override = kstrndup(buf, len, GFP_KERNEL); + if (!driver_override) + return -ENOMEM; + + device_lock(dev); + old = spi->driver_override; + if (len) { + spi->driver_override = driver_override; + } else { + /* Emptry string, disable driver override */ + spi->driver_override = NULL; + kfree(driver_override); + } + device_unlock(dev); + kfree(old); + + return count; +} + +static ssize_t driver_override_show(struct device *dev, + struct device_attribute *a, char *buf) +{ + const struct spi_device *spi = to_spi_device(dev); + ssize_t len; + + device_lock(dev); + len = snprintf(buf, PAGE_SIZE, "%s\n", spi->driver_override ? : ""); + device_unlock(dev); + return len; +} +static DEVICE_ATTR_RW(driver_override); + #define SPI_STATISTICS_ATTRS(field, file) \ static ssize_t spi_controller_##field##_show(struct device *dev, \ struct device_attribute *attr, \ @@ -158,6 +195,7 @@ SPI_STATISTICS_SHOW(transfers_split_maxsize, "%lu"); static struct attribute *spi_dev_attrs[] = { &dev_attr_modalias.attr, + &dev_attr_driver_override.attr, NULL, }; @@ -305,6 +343,10 @@ static int spi_match_device(struct device *dev, struct device_driver *drv) const struct spi_device *spi = to_spi_device(dev); const struct spi_driver *sdrv = to_spi_driver(drv); + /* Check override first, and if set, only use the named driver */ + if (spi->driver_override) + return strcmp(spi->driver_override, drv->name) == 0; + /* Attempt an OF style match */ if (of_driver_match_device(dev, drv)) return 1; @@ -733,7 +775,9 @@ static void spi_set_cs(struct spi_device *spi, bool enable) enable = !enable; if (gpio_is_valid(spi->cs_gpio)) { - gpio_set_value(spi->cs_gpio, !enable); + /* Honour the SPI_NO_CS flag */ + if (!(spi->mode & SPI_NO_CS)) + gpio_set_value(spi->cs_gpio, !enable); /* Some SPI masters need both GPIO CS & slave_select */ if ((spi->controller->flags & SPI_MASTER_GPIO_SS) && spi->controller->set_cs) @@ -2783,8 +2827,10 @@ int spi_setup(struct spi_device *spi) return -EINVAL; /* help drivers fail *cleanly* when they need options * that aren't supported with their current controller + * SPI_CS_WORD has a fallback software implementation, + * so it is ignored here. */ - bad_bits = spi->mode & ~spi->controller->mode_bits; + bad_bits = spi->mode & ~(spi->controller->mode_bits | SPI_CS_WORD); ugly_bits = bad_bits & (SPI_TX_DUAL | SPI_TX_QUAD | SPI_RX_DUAL | SPI_RX_QUAD); if (ugly_bits) { @@ -2838,6 +2884,35 @@ static int __spi_validate(struct spi_device *spi, struct spi_message *message) if (list_empty(&message->transfers)) return -EINVAL; + /* If an SPI controller does not support toggling the CS line on each + * transfer (indicated by the SPI_CS_WORD flag) or we are using a GPIO + * for the CS line, we can emulate the CS-per-word hardware function by + * splitting transfers into one-word transfers and ensuring that + * cs_change is set for each transfer. + */ + if ((spi->mode & SPI_CS_WORD) && (!(ctlr->mode_bits & SPI_CS_WORD) || + gpio_is_valid(spi->cs_gpio))) { + size_t maxsize; + int ret; + + maxsize = (spi->bits_per_word + 7) / 8; + + /* spi_split_transfers_maxsize() requires message->spi */ + message->spi = spi; + + ret = spi_split_transfers_maxsize(ctlr, message, maxsize, + GFP_KERNEL); + if (ret) + return ret; + + list_for_each_entry(xfer, &message->transfers, transfer_list) { + /* don't change cs_change on the last entry in the list */ + if (list_is_last(&xfer->transfer_list, &message->transfers)) + break; + xfer->cs_change = 1; + } + } + /* Half-duplex links include original MicroWire, and ones with * only one data pin like SPI_3WIRE (switches direction) or where * either MOSI or MISO is missing. They can also be caused by @@ -3323,20 +3398,23 @@ EXPORT_SYMBOL_GPL(spi_write_then_read); /*-------------------------------------------------------------------------*/ -#if IS_ENABLED(CONFIG_OF_DYNAMIC) +#if IS_ENABLED(CONFIG_OF) static int __spi_of_device_match(struct device *dev, void *data) { return dev->of_node == data; } /* must call put_device() when done with returned spi_device device */ -static struct spi_device *of_find_spi_device_by_node(struct device_node *node) +struct spi_device *of_find_spi_device_by_node(struct device_node *node) { struct device *dev = bus_find_device(&spi_bus_type, NULL, node, __spi_of_device_match); return dev ? to_spi_device(dev) : NULL; } +EXPORT_SYMBOL_GPL(of_find_spi_device_by_node); +#endif /* IS_ENABLED(CONFIG_OF) */ +#if IS_ENABLED(CONFIG_OF_DYNAMIC) static int __spi_of_controller_match(struct device *dev, const void *data) { return dev->of_node == data; diff --git a/drivers/spi/spidev.c b/drivers/spi/spidev.c index cda10719d1d1..b0c76e2626ce 100644 --- a/drivers/spi/spidev.c +++ b/drivers/spi/spidev.c @@ -669,6 +669,7 @@ static const struct of_device_id spidev_dt_ids[] = { { .compatible = "lineartechnology,ltc2488" }, { .compatible = "ge,achc" }, { .compatible = "semtech,sx1301" }, + { .compatible = "lwn,bk4" }, {}, }; MODULE_DEVICE_TABLE(of, spidev_dt_ids); @@ -724,11 +725,9 @@ static int spidev_probe(struct spi_device *spi) * compatible string, it is a Linux implementation thing * rather than a description of the hardware. */ - if (spi->dev.of_node && !of_match_device(spidev_dt_ids, &spi->dev)) { - dev_err(&spi->dev, "buggy DT: spidev listed directly in DT\n"); - WARN_ON(spi->dev.of_node && - !of_match_device(spidev_dt_ids, &spi->dev)); - } + WARN(spi->dev.of_node && + of_device_is_compatible(spi->dev.of_node, "spidev"), + "%pOF: buggy DT: spidev listed directly in DT\n", spi->dev.of_node); spidev_probe_acpi(spi); diff --git a/drivers/staging/mt29f_spinand/mt29f_spinand.c b/drivers/staging/mt29f_spinand/mt29f_spinand.c index 448478451c4c..def8a1f57d1c 100644 --- a/drivers/staging/mt29f_spinand/mt29f_spinand.c +++ b/drivers/staging/mt29f_spinand/mt29f_spinand.c @@ -630,8 +630,7 @@ static int spinand_erase_block(struct spi_device *spi_nand, u16 block_id) } #ifdef CONFIG_MTD_SPINAND_ONDIEECC -static int spinand_write_page_hwecc(struct mtd_info *mtd, - struct nand_chip *chip, +static int spinand_write_page_hwecc(struct nand_chip *chip, const u8 *buf, int oob_required, int page) { @@ -643,21 +642,22 @@ static int spinand_write_page_hwecc(struct mtd_info *mtd, return nand_prog_page_op(chip, page, 0, p, eccsize * eccsteps); } -static int spinand_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, - u8 *buf, int oob_required, int page) +static int spinand_read_page_hwecc(struct nand_chip *chip, u8 *buf, + int oob_required, int page) { int retval; u8 status; u8 *p = buf; int eccsize = chip->ecc.size; int eccsteps = chip->ecc.steps; + struct mtd_info *mtd = nand_to_mtd(chip); struct spinand_info *info = nand_get_controller_data(chip); enable_read_hw_ecc = 1; nand_read_page_op(chip, page, 0, p, eccsize * eccsteps); if (oob_required) - chip->read_buf(mtd, chip->oob_poi, mtd->oobsize); + chip->legacy.read_buf(chip, chip->oob_poi, mtd->oobsize); while (1) { retval = spinand_read_status(info->spi, &status); @@ -681,13 +681,13 @@ static int spinand_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, } #endif -static void spinand_select_chip(struct mtd_info *mtd, int dev) +static void spinand_select_chip(struct nand_chip *chip, int dev) { } -static u8 spinand_read_byte(struct mtd_info *mtd) +static u8 spinand_read_byte(struct nand_chip *chip) { - struct spinand_state *state = mtd_to_state(mtd); + struct spinand_state *state = mtd_to_state(nand_to_mtd(chip)); u8 data; data = state->buf[state->buf_ptr]; @@ -695,8 +695,9 @@ static u8 spinand_read_byte(struct mtd_info *mtd) return data; } -static int spinand_wait(struct mtd_info *mtd, struct nand_chip *chip) +static int spinand_wait(struct nand_chip *chip) { + struct mtd_info *mtd = nand_to_mtd(chip); struct spinand_info *info = nand_get_controller_data(chip); unsigned long timeo = jiffies; @@ -724,17 +725,17 @@ static int spinand_wait(struct mtd_info *mtd, struct nand_chip *chip) return 0; } -static void spinand_write_buf(struct mtd_info *mtd, const u8 *buf, int len) +static void spinand_write_buf(struct nand_chip *chip, const u8 *buf, int len) { - struct spinand_state *state = mtd_to_state(mtd); + struct spinand_state *state = mtd_to_state(nand_to_mtd(chip)); memcpy(state->buf + state->buf_ptr, buf, len); state->buf_ptr += len; } -static void spinand_read_buf(struct mtd_info *mtd, u8 *buf, int len) +static void spinand_read_buf(struct nand_chip *chip, u8 *buf, int len) { - struct spinand_state *state = mtd_to_state(mtd); + struct spinand_state *state = mtd_to_state(nand_to_mtd(chip)); memcpy(buf, state->buf + state->buf_ptr, len); state->buf_ptr += len; @@ -759,10 +760,10 @@ static void spinand_reset(struct spi_device *spi_nand) dev_err(&spi_nand->dev, "wait timedout!\n"); } -static void spinand_cmdfunc(struct mtd_info *mtd, unsigned int command, +static void spinand_cmdfunc(struct nand_chip *chip, unsigned int command, int column, int page) { - struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(chip); struct spinand_info *info = nand_get_controller_data(chip); struct spinand_state *state = info->priv; @@ -914,15 +915,15 @@ static int spinand_probe(struct spi_device *spi_nand) nand_set_flash_node(chip, spi_nand->dev.of_node); nand_set_controller_data(chip, info); - chip->read_buf = spinand_read_buf; - chip->write_buf = spinand_write_buf; - chip->read_byte = spinand_read_byte; - chip->cmdfunc = spinand_cmdfunc; - chip->waitfunc = spinand_wait; + chip->legacy.read_buf = spinand_read_buf; + chip->legacy.write_buf = spinand_write_buf; + chip->legacy.read_byte = spinand_read_byte; + chip->legacy.cmdfunc = spinand_cmdfunc; + chip->legacy.waitfunc = spinand_wait; chip->options |= NAND_CACHEPRG; chip->select_chip = spinand_select_chip; - chip->set_features = nand_get_set_features_notsupp; - chip->get_features = nand_get_set_features_notsupp; + chip->legacy.set_features = nand_get_set_features_notsupp; + chip->legacy.get_features = nand_get_set_features_notsupp; mtd = nand_to_mtd(chip); @@ -934,7 +935,7 @@ static int spinand_probe(struct spi_device *spi_nand) mtd_set_ooblayout(mtd, &spinand_oob_64_ops); #endif - if (nand_scan(mtd, 1)) + if (nand_scan(chip, 1)) return -ENXIO; return mtd_device_register(mtd, NULL, 0); diff --git a/drivers/target/target_core_spc.c b/drivers/target/target_core_spc.c index cb0461a10808..f459118bc11b 100644 --- a/drivers/target/target_core_spc.c +++ b/drivers/target/target_core_spc.c @@ -636,9 +636,9 @@ spc_emulate_evpd_b2(struct se_cmd *cmd, unsigned char *buf) /* * The unmap_zeroes_data set means that the underlying device supports - * REQ_DISCARD and has the discard_zeroes_data bit set. This satisfies - * the SBC requirements for LBPRZ, meaning that a subsequent read - * will return zeroes after an UNMAP or WRITE SAME (16) to an LBA + * REQ_OP_DISCARD and has the discard_zeroes_data bit set. This + * satisfies the SBC requirements for LBPRZ, meaning that a subsequent + * read will return zeroes after an UNMAP or WRITE SAME (16) to an LBA * See sbc4r36 6.6.4. */ if (((dev->dev_attrib.emulate_tpu != 0) || diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c index bc03b0a690b4..9ede35cecb12 100644 --- a/drivers/usb/class/cdc-acm.c +++ b/drivers/usb/class/cdc-acm.c @@ -310,17 +310,17 @@ static void acm_process_notification(struct acm *acm, unsigned char *buf) if (difference & ACM_CTRL_DSR) acm->iocount.dsr++; - if (difference & ACM_CTRL_BRK) - acm->iocount.brk++; - if (difference & ACM_CTRL_RI) - acm->iocount.rng++; if (difference & ACM_CTRL_DCD) acm->iocount.dcd++; - if (difference & ACM_CTRL_FRAMING) + if (newctrl & ACM_CTRL_BRK) + acm->iocount.brk++; + if (newctrl & ACM_CTRL_RI) + acm->iocount.rng++; + if (newctrl & ACM_CTRL_FRAMING) acm->iocount.frame++; - if (difference & ACM_CTRL_PARITY) + if (newctrl & ACM_CTRL_PARITY) acm->iocount.parity++; - if (difference & ACM_CTRL_OVERRUN) + if (newctrl & ACM_CTRL_OVERRUN) acm->iocount.overrun++; spin_unlock_irqrestore(&acm->read_lock, flags); @@ -355,7 +355,6 @@ static void acm_ctrl_irq(struct urb *urb) case -ENOENT: case -ESHUTDOWN: /* this urb is terminated, clean up */ - acm->nb_index = 0; dev_dbg(&acm->control->dev, "%s - urb shutting down with status: %d\n", __func__, status); @@ -1642,6 +1641,7 @@ static int acm_pre_reset(struct usb_interface *intf) struct acm *acm = usb_get_intfdata(intf); clear_bit(EVENT_RX_STALL, &acm->flags); + acm->nb_index = 0; /* pending control transfers are lost */ return 0; } diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c index 244417d0dfd1..ffccd40ea67d 100644 --- a/drivers/usb/core/devio.c +++ b/drivers/usb/core/devio.c @@ -1474,8 +1474,6 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb u = 0; switch (uurb->type) { case USBDEVFS_URB_TYPE_CONTROL: - if (is_in) - allow_short = true; if (!usb_endpoint_xfer_control(&ep->desc)) return -EINVAL; /* min 8 byte setup packet */ @@ -1505,6 +1503,8 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb is_in = 0; uurb->endpoint &= ~USB_DIR_IN; } + if (is_in) + allow_short = true; snoop(&ps->dev->dev, "control urb: bRequestType=%02x " "bRequest=%02x wValue=%04x " "wIndex=%04x wLength=%04x\n", diff --git a/drivers/usb/gadget/function/f_mass_storage.c b/drivers/usb/gadget/function/f_mass_storage.c index ca8a4b53c59f..1074cb82ec17 100644 --- a/drivers/usb/gadget/function/f_mass_storage.c +++ b/drivers/usb/gadget/function/f_mass_storage.c @@ -221,6 +221,8 @@ #include <linux/usb/gadget.h> #include <linux/usb/composite.h> +#include <linux/nospec.h> + #include "configfs.h" @@ -3152,6 +3154,7 @@ static struct config_group *fsg_lun_make(struct config_group *group, fsg_opts = to_fsg_opts(&group->cg_item); if (num >= FSG_MAX_LUNS) return ERR_PTR(-ERANGE); + num = array_index_nospec(num, FSG_MAX_LUNS); mutex_lock(&fsg_opts->lock); if (fsg_opts->refcnt || fsg_opts->common->luns[num]) { diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index 722860eb5a91..51dd8e00c4f8 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -179,10 +179,12 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) xhci->quirks |= XHCI_PME_STUCK_QUIRK; } if (pdev->vendor == PCI_VENDOR_ID_INTEL && - pdev->device == PCI_DEVICE_ID_INTEL_CHERRYVIEW_XHCI) { + pdev->device == PCI_DEVICE_ID_INTEL_CHERRYVIEW_XHCI) xhci->quirks |= XHCI_SSIC_PORT_UNUSED; + if (pdev->vendor == PCI_VENDOR_ID_INTEL && + (pdev->device == PCI_DEVICE_ID_INTEL_CHERRYVIEW_XHCI || + pdev->device == PCI_DEVICE_ID_INTEL_APL_XHCI)) xhci->quirks |= XHCI_INTEL_USB_ROLE_SW; - } if (pdev->vendor == PCI_VENDOR_ID_INTEL && (pdev->device == PCI_DEVICE_ID_INTEL_CHERRYVIEW_XHCI || pdev->device == PCI_DEVICE_ID_INTEL_SUNRISEPOINT_LP_XHCI || diff --git a/drivers/usb/roles/intel-xhci-usb-role-switch.c b/drivers/usb/roles/intel-xhci-usb-role-switch.c index 1fb3dd0f1dfa..277de96181f9 100644 --- a/drivers/usb/roles/intel-xhci-usb-role-switch.c +++ b/drivers/usb/roles/intel-xhci-usb-role-switch.c @@ -161,6 +161,8 @@ static int intel_xhci_usb_remove(struct platform_device *pdev) { struct intel_xhci_usb_data *data = platform_get_drvdata(pdev); + pm_runtime_disable(&pdev->dev); + usb_role_switch_unregister(data->role_sw); return 0; } diff --git a/drivers/usb/usbip/vhci_hcd.c b/drivers/usb/usbip/vhci_hcd.c index d11f3f8dad40..1e592ec94ba4 100644 --- a/drivers/usb/usbip/vhci_hcd.c +++ b/drivers/usb/usbip/vhci_hcd.c @@ -318,8 +318,9 @@ static int vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, struct vhci_hcd *vhci_hcd; struct vhci *vhci; int retval = 0; - int rhport; + int rhport = -1; unsigned long flags; + bool invalid_rhport = false; u32 prev_port_status[VHCI_HC_PORTS]; @@ -334,9 +335,19 @@ static int vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, usbip_dbg_vhci_rh("typeReq %x wValue %x wIndex %x\n", typeReq, wValue, wIndex); - if (wIndex > VHCI_HC_PORTS) - pr_err("invalid port number %d\n", wIndex); - rhport = wIndex - 1; + /* + * wIndex can be 0 for some request types (typeReq). rhport is + * in valid range when wIndex >= 1 and < VHCI_HC_PORTS. + * + * Reference port_status[] only with valid rhport when + * invalid_rhport is false. + */ + if (wIndex < 1 || wIndex > VHCI_HC_PORTS) { + invalid_rhport = true; + if (wIndex > VHCI_HC_PORTS) + pr_err("invalid port number %d\n", wIndex); + } else + rhport = wIndex - 1; vhci_hcd = hcd_to_vhci_hcd(hcd); vhci = vhci_hcd->vhci; @@ -345,8 +356,9 @@ static int vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, /* store old status and compare now and old later */ if (usbip_dbg_flag_vhci_rh) { - memcpy(prev_port_status, vhci_hcd->port_status, - sizeof(prev_port_status)); + if (!invalid_rhport) + memcpy(prev_port_status, vhci_hcd->port_status, + sizeof(prev_port_status)); } switch (typeReq) { @@ -354,8 +366,10 @@ static int vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, usbip_dbg_vhci_rh(" ClearHubFeature\n"); break; case ClearPortFeature: - if (rhport < 0) + if (invalid_rhport) { + pr_err("invalid port number %d\n", wIndex); goto error; + } switch (wValue) { case USB_PORT_FEAT_SUSPEND: if (hcd->speed == HCD_USB3) { @@ -415,9 +429,10 @@ static int vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, break; case GetPortStatus: usbip_dbg_vhci_rh(" GetPortStatus port %x\n", wIndex); - if (wIndex < 1) { + if (invalid_rhport) { pr_err("invalid port number %d\n", wIndex); retval = -EPIPE; + goto error; } /* we do not care about resume. */ @@ -513,16 +528,20 @@ static int vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, goto error; } - if (rhport < 0) + if (invalid_rhport) { + pr_err("invalid port number %d\n", wIndex); goto error; + } vhci_hcd->port_status[rhport] |= USB_PORT_STAT_SUSPEND; break; case USB_PORT_FEAT_POWER: usbip_dbg_vhci_rh( " SetPortFeature: USB_PORT_FEAT_POWER\n"); - if (rhport < 0) + if (invalid_rhport) { + pr_err("invalid port number %d\n", wIndex); goto error; + } if (hcd->speed == HCD_USB3) vhci_hcd->port_status[rhport] |= USB_SS_PORT_STAT_POWER; else @@ -531,8 +550,10 @@ static int vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, case USB_PORT_FEAT_BH_PORT_RESET: usbip_dbg_vhci_rh( " SetPortFeature: USB_PORT_FEAT_BH_PORT_RESET\n"); - if (rhport < 0) + if (invalid_rhport) { + pr_err("invalid port number %d\n", wIndex); goto error; + } /* Applicable only for USB3.0 hub */ if (hcd->speed != HCD_USB3) { pr_err("USB_PORT_FEAT_BH_PORT_RESET req not " @@ -543,8 +564,10 @@ static int vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, case USB_PORT_FEAT_RESET: usbip_dbg_vhci_rh( " SetPortFeature: USB_PORT_FEAT_RESET\n"); - if (rhport < 0) + if (invalid_rhport) { + pr_err("invalid port number %d\n", wIndex); goto error; + } /* if it's already enabled, disable */ if (hcd->speed == HCD_USB3) { vhci_hcd->port_status[rhport] = 0; @@ -565,8 +588,10 @@ static int vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, default: usbip_dbg_vhci_rh(" SetPortFeature: default %d\n", wValue); - if (rhport < 0) + if (invalid_rhport) { + pr_err("invalid port number %d\n", wIndex); goto error; + } if (hcd->speed == HCD_USB3) { if ((vhci_hcd->port_status[rhport] & USB_SS_PORT_STAT_POWER) != 0) { @@ -608,7 +633,7 @@ error: if (usbip_dbg_flag_vhci_rh) { pr_debug("port %d\n", rhport); /* Only dump valid port status */ - if (rhport >= 0) { + if (!invalid_rhport) { dump_port_status_diff(prev_port_status[rhport], vhci_hcd->port_status[rhport], hcd->speed == HCD_USB3); @@ -618,8 +643,10 @@ error: spin_unlock_irqrestore(&vhci->lock, flags); - if ((vhci_hcd->port_status[rhport] & PORT_C_MASK) != 0) + if (!invalid_rhport && + (vhci_hcd->port_status[rhport] & PORT_C_MASK) != 0) { usb_hcd_poll_rh_status(hcd); + } return retval; } diff --git a/drivers/xen/biomerge.c b/drivers/xen/biomerge.c index 55ed80c3a17c..f3fbb700f569 100644 --- a/drivers/xen/biomerge.c +++ b/drivers/xen/biomerge.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/bio.h> -#include <linux/io.h> #include <linux/export.h> +#include <xen/xen.h> #include <xen/page.h> bool xen_biovec_phys_mergeable(const struct bio_vec *vec1, @@ -20,4 +20,3 @@ bool xen_biovec_phys_mergeable(const struct bio_vec *vec1, return false; #endif } -EXPORT_SYMBOL(xen_biovec_phys_mergeable); diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index a6f9ba85dc4b..28819a0e61d0 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -662,7 +662,7 @@ xen_swiotlb_dma_mmap(struct device *dev, struct vm_area_struct *vma, return xen_get_dma_ops(dev)->mmap(dev, vma, cpu_addr, dma_addr, size, attrs); #endif - return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size); + return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); } /* @@ -689,7 +689,7 @@ xen_swiotlb_get_sgtable(struct device *dev, struct sg_table *sgt, handle, size, attrs); } #endif - return dma_common_get_sgtable(dev, sgt, cpu_addr, handle, size); + return dma_common_get_sgtable(dev, sgt, cpu_addr, handle, size, attrs); } static int xen_swiotlb_mapping_error(struct device *dev, dma_addr_t dma_addr) diff --git a/drivers/xen/xen-acpi-pad.c b/drivers/xen/xen-acpi-pad.c index 23d1808fe027..e25ab76b9c99 100644 --- a/drivers/xen/xen-acpi-pad.c +++ b/drivers/xen/xen-acpi-pad.c @@ -19,6 +19,7 @@ #include <linux/kernel.h> #include <linux/types.h> #include <linux/acpi.h> +#include <xen/xen.h> #include <xen/interface/version.h> #include <xen/xen-ops.h> #include <asm/xen/hypercall.h> diff --git a/fs/afs/rxrpc.c b/fs/afs/rxrpc.c index 35f2ae30f31f..77a83790a31f 100644 --- a/fs/afs/rxrpc.c +++ b/fs/afs/rxrpc.c @@ -690,8 +690,6 @@ static void afs_process_async_call(struct work_struct *work) } if (call->state == AFS_CALL_COMPLETE) { - call->reply[0] = NULL; - /* We have two refs to release - one from the alloc and one * queued with the work item - and we can't just deallocate the * call because the work item may be queued again. diff --git a/fs/afs/server.c b/fs/afs/server.c index 2f306c0cc4ee..1d329e6981d5 100644 --- a/fs/afs/server.c +++ b/fs/afs/server.c @@ -199,11 +199,9 @@ static struct afs_server *afs_install_server(struct afs_net *net, write_sequnlock(&net->fs_addr_lock); ret = 0; - goto out; exists: afs_get_server(server); -out: write_sequnlock(&net->fs_lock); return server; } diff --git a/fs/buffer.c b/fs/buffer.c index 6f1ae3ac9789..109f55196866 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -3060,11 +3060,6 @@ static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh, */ bio = bio_alloc(GFP_NOIO, 1); - if (wbc) { - wbc_init_bio(wbc, bio); - wbc_account_io(wbc, bh->b_page, bh->b_size); - } - bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); bio_set_dev(bio, bh->b_bdev); bio->bi_write_hint = write_hint; @@ -3084,6 +3079,11 @@ static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh, op_flags |= REQ_PRIO; bio_set_op_attrs(bio, op, op_flags); + if (wbc) { + wbc_init_bio(wbc, bio); + wbc_account_io(wbc, bh->b_page, bh->b_size); + } + submit_bio(bio); return 0; } diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c index af2b17b21b94..95983c744164 100644 --- a/fs/cachefiles/namei.c +++ b/fs/cachefiles/namei.c @@ -343,7 +343,7 @@ try_again: trap = lock_rename(cache->graveyard, dir); /* do some checks before getting the grave dentry */ - if (rep->d_parent != dir) { + if (rep->d_parent != dir || IS_DEADDIR(d_inode(rep))) { /* the entry was probably culled when we dropped the parent dir * lock */ unlock_rename(cache->graveyard, dir); diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index db7590178dfc..2aa62d58d8dd 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -374,13 +374,13 @@ static int io_submit_init_bio(struct ext4_io_submit *io, bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); if (!bio) return -ENOMEM; - wbc_init_bio(io->io_wbc, bio); bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); bio_set_dev(bio, bh->b_bdev); bio->bi_end_io = ext4_end_bio; bio->bi_private = ext4_get_io_end(io->io_end); io->io_bio = bio; io->io_next_block = bh->b_blocknr; + wbc_init_bio(io->io_wbc, bio); return 0; } diff --git a/fs/fscache/cookie.c b/fs/fscache/cookie.c index 83bfe04456b6..c550512ce335 100644 --- a/fs/fscache/cookie.c +++ b/fs/fscache/cookie.c @@ -70,20 +70,7 @@ void fscache_free_cookie(struct fscache_cookie *cookie) } /* - * initialise an cookie jar slab element prior to any use - */ -void fscache_cookie_init_once(void *_cookie) -{ - struct fscache_cookie *cookie = _cookie; - - memset(cookie, 0, sizeof(*cookie)); - spin_lock_init(&cookie->lock); - spin_lock_init(&cookie->stores_lock); - INIT_HLIST_HEAD(&cookie->backing_objects); -} - -/* - * Set the index key in a cookie. The cookie struct has space for a 12-byte + * Set the index key in a cookie. The cookie struct has space for a 16-byte * key plus length and hash, but if that's not big enough, it's instead a * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then * the key data. @@ -93,20 +80,18 @@ static int fscache_set_key(struct fscache_cookie *cookie, { unsigned long long h; u32 *buf; + int bufs; int i; - cookie->key_len = index_key_len; + bufs = DIV_ROUND_UP(index_key_len, sizeof(*buf)); if (index_key_len > sizeof(cookie->inline_key)) { - buf = kzalloc(index_key_len, GFP_KERNEL); + buf = kcalloc(bufs, sizeof(*buf), GFP_KERNEL); if (!buf) return -ENOMEM; cookie->key = buf; } else { buf = (u32 *)cookie->inline_key; - buf[0] = 0; - buf[1] = 0; - buf[2] = 0; } memcpy(buf, index_key, index_key_len); @@ -116,7 +101,8 @@ static int fscache_set_key(struct fscache_cookie *cookie, */ h = (unsigned long)cookie->parent; h += index_key_len + cookie->type; - for (i = 0; i < (index_key_len + sizeof(u32) - 1) / sizeof(u32); i++) + + for (i = 0; i < bufs; i++) h += buf[i]; cookie->key_hash = h ^ (h >> 32); @@ -161,7 +147,7 @@ struct fscache_cookie *fscache_alloc_cookie( struct fscache_cookie *cookie; /* allocate and initialise a cookie */ - cookie = kmem_cache_alloc(fscache_cookie_jar, GFP_KERNEL); + cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL); if (!cookie) return NULL; @@ -192,6 +178,9 @@ struct fscache_cookie *fscache_alloc_cookie( cookie->netfs_data = netfs_data; cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET); cookie->type = def->type; + spin_lock_init(&cookie->lock); + spin_lock_init(&cookie->stores_lock); + INIT_HLIST_HEAD(&cookie->backing_objects); /* radix tree insertion won't use the preallocation pool unless it's * told it may not wait */ diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h index f83328a7f048..d6209022e965 100644 --- a/fs/fscache/internal.h +++ b/fs/fscache/internal.h @@ -51,7 +51,6 @@ extern struct fscache_cache *fscache_select_cache_for_object( extern struct kmem_cache *fscache_cookie_jar; extern void fscache_free_cookie(struct fscache_cookie *); -extern void fscache_cookie_init_once(void *); extern struct fscache_cookie *fscache_alloc_cookie(struct fscache_cookie *, const struct fscache_cookie_def *, const void *, size_t, diff --git a/fs/fscache/main.c b/fs/fscache/main.c index 7dce110bf17d..30ad89db1efc 100644 --- a/fs/fscache/main.c +++ b/fs/fscache/main.c @@ -143,9 +143,7 @@ static int __init fscache_init(void) fscache_cookie_jar = kmem_cache_create("fscache_cookie_jar", sizeof(struct fscache_cookie), - 0, - 0, - fscache_cookie_init_once); + 0, 0, NULL); if (!fscache_cookie_jar) { pr_notice("Failed to allocate a cookie jar\n"); ret = -ENOMEM; diff --git a/fs/jffs2/super.c b/fs/jffs2/super.c index 87bdf0f4cba1..902a7dd10e5c 100644 --- a/fs/jffs2/super.c +++ b/fs/jffs2/super.c @@ -285,10 +285,8 @@ static int jffs2_fill_super(struct super_block *sb, void *data, int silent) sb->s_fs_info = c; ret = jffs2_parse_options(c, data); - if (ret) { - kfree(c); + if (ret) return -EINVAL; - } /* Initialize JFFS2 superblock locks, the further initialization will * be done later */ diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h index ba4dd54f2c82..53600f527a70 100644 --- a/include/acpi/acpi_bus.h +++ b/include/acpi/acpi_bus.h @@ -595,7 +595,6 @@ enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev); int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset, u64 *size); int acpi_dma_configure(struct device *dev, enum dev_dma_attr attr); -void acpi_dma_deconfigure(struct device *dev); struct acpi_device *acpi_find_child_device(struct acpi_device *parent, u64 address, bool check_children); diff --git a/include/asm-generic/dma-mapping.h b/include/asm-generic/dma-mapping.h index ad2868263867..880a292d792f 100644 --- a/include/asm-generic/dma-mapping.h +++ b/include/asm-generic/dma-mapping.h @@ -4,16 +4,7 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) { - /* - * Use the non-coherent ops if available. If an architecture wants a - * more fine-grained selection of operations it will have to implement - * get_arch_dma_ops itself or use the per-device dma_ops. - */ -#ifdef CONFIG_DMA_NONCOHERENT_OPS - return &dma_noncoherent_ops; -#else return &dma_direct_ops; -#endif } #endif /* _ASM_GENERIC_DMA_MAPPING_H */ diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index b3353e21f3b3..6be86c1c5c58 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -20,6 +20,8 @@ #include <asm/pgalloc.h> #include <asm/tlbflush.h> +#ifdef CONFIG_MMU + #ifdef CONFIG_HAVE_RCU_TABLE_FREE /* * Semi RCU freeing of the page directories. @@ -97,12 +99,30 @@ struct mmu_gather { #endif unsigned long start; unsigned long end; - /* we are in the middle of an operation to clear - * a full mm and can make some optimizations */ - unsigned int fullmm : 1, - /* we have performed an operation which - * requires a complete flush of the tlb */ - need_flush_all : 1; + /* + * we are in the middle of an operation to clear + * a full mm and can make some optimizations + */ + unsigned int fullmm : 1; + + /* + * we have performed an operation which + * requires a complete flush of the tlb + */ + unsigned int need_flush_all : 1; + + /* + * we have removed page directories + */ + unsigned int freed_tables : 1; + + /* + * at which levels have we cleared entries? + */ + unsigned int cleared_ptes : 1; + unsigned int cleared_pmds : 1; + unsigned int cleared_puds : 1; + unsigned int cleared_p4ds : 1; struct mmu_gather_batch *active; struct mmu_gather_batch local; @@ -118,6 +138,7 @@ void arch_tlb_gather_mmu(struct mmu_gather *tlb, void tlb_flush_mmu(struct mmu_gather *tlb); void arch_tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end, bool force); +void tlb_flush_mmu_free(struct mmu_gather *tlb); extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size); @@ -137,6 +158,11 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb) tlb->start = TASK_SIZE; tlb->end = 0; } + tlb->freed_tables = 0; + tlb->cleared_ptes = 0; + tlb->cleared_pmds = 0; + tlb->cleared_puds = 0; + tlb->cleared_p4ds = 0; } static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) @@ -186,6 +212,25 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, } #endif +static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb) +{ + if (tlb->cleared_ptes) + return PAGE_SHIFT; + if (tlb->cleared_pmds) + return PMD_SHIFT; + if (tlb->cleared_puds) + return PUD_SHIFT; + if (tlb->cleared_p4ds) + return P4D_SHIFT; + + return PAGE_SHIFT; +} + +static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) +{ + return 1UL << tlb_get_unmap_shift(tlb); +} + /* * In the case of tlb vma handling, we can optimise these away in the * case where we're doing a full MM flush. When we're doing a munmap, @@ -219,13 +264,19 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define tlb_remove_tlb_entry(tlb, ptep, address) \ do { \ __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + tlb->cleared_ptes = 1; \ __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) -#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \ - do { \ - __tlb_adjust_range(tlb, address, huge_page_size(h)); \ - __tlb_remove_tlb_entry(tlb, ptep, address); \ +#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \ + do { \ + unsigned long _sz = huge_page_size(h); \ + __tlb_adjust_range(tlb, address, _sz); \ + if (_sz == PMD_SIZE) \ + tlb->cleared_pmds = 1; \ + else if (_sz == PUD_SIZE) \ + tlb->cleared_puds = 1; \ + __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) /** @@ -239,6 +290,7 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \ do { \ __tlb_adjust_range(tlb, address, HPAGE_PMD_SIZE); \ + tlb->cleared_pmds = 1; \ __tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \ } while (0) @@ -253,6 +305,7 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define tlb_remove_pud_tlb_entry(tlb, pudp, address) \ do { \ __tlb_adjust_range(tlb, address, HPAGE_PUD_SIZE); \ + tlb->cleared_puds = 1; \ __tlb_remove_pud_tlb_entry(tlb, pudp, address); \ } while (0) @@ -278,6 +331,8 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define pte_free_tlb(tlb, ptep, address) \ do { \ __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + tlb->freed_tables = 1; \ + tlb->cleared_pmds = 1; \ __pte_free_tlb(tlb, ptep, address); \ } while (0) #endif @@ -285,7 +340,9 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #ifndef pmd_free_tlb #define pmd_free_tlb(tlb, pmdp, address) \ do { \ - __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + tlb->freed_tables = 1; \ + tlb->cleared_puds = 1; \ __pmd_free_tlb(tlb, pmdp, address); \ } while (0) #endif @@ -295,6 +352,8 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define pud_free_tlb(tlb, pudp, address) \ do { \ __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + tlb->freed_tables = 1; \ + tlb->cleared_p4ds = 1; \ __pud_free_tlb(tlb, pudp, address); \ } while (0) #endif @@ -304,12 +363,15 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #ifndef p4d_free_tlb #define p4d_free_tlb(tlb, pudp, address) \ do { \ - __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + tlb->freed_tables = 1; \ __p4d_free_tlb(tlb, pudp, address); \ } while (0) #endif #endif +#endif /* CONFIG_MMU */ + #define tlb_migrate_finish(mm) do {} while (0) #endif /* _ASM_GENERIC__TLB_H */ diff --git a/include/drm/drm_atomic.h b/include/drm/drm_atomic.h index da9d95a19580..1e713154f00e 100644 --- a/include/drm/drm_atomic.h +++ b/include/drm/drm_atomic.h @@ -153,6 +153,17 @@ struct __drm_planes_state { struct __drm_crtcs_state { struct drm_crtc *ptr; struct drm_crtc_state *state, *old_state, *new_state; + + /** + * @commit: + * + * A reference to the CRTC commit object that is kept for use by + * drm_atomic_helper_wait_for_flip_done() after + * drm_atomic_helper_commit_hw_done() is called. This ensures that a + * concurrent commit won't free a commit object that is still in use. + */ + struct drm_crtc_commit *commit; + s32 __user *out_fence_ptr; u64 last_vblank_count; }; diff --git a/include/drm/drm_edid.h b/include/drm/drm_edid.h index b25d12ef120a..e3c404833115 100644 --- a/include/drm/drm_edid.h +++ b/include/drm/drm_edid.h @@ -214,9 +214,9 @@ struct detailed_timing { #define DRM_EDID_HDMI_DC_Y444 (1 << 3) /* YCBCR 420 deep color modes */ -#define DRM_EDID_YCBCR420_DC_48 (1 << 6) -#define DRM_EDID_YCBCR420_DC_36 (1 << 5) -#define DRM_EDID_YCBCR420_DC_30 (1 << 4) +#define DRM_EDID_YCBCR420_DC_48 (1 << 2) +#define DRM_EDID_YCBCR420_DC_36 (1 << 1) +#define DRM_EDID_YCBCR420_DC_30 (1 << 0) #define DRM_EDID_YCBCR420_DC_MASK (DRM_EDID_YCBCR420_DC_48 | \ DRM_EDID_YCBCR420_DC_36 | \ DRM_EDID_YCBCR420_DC_30) diff --git a/include/linux/acpi.h b/include/linux/acpi.h index de8d3d3fa651..af4628979d13 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -831,8 +831,6 @@ static inline int acpi_dma_configure(struct device *dev, return 0; } -static inline void acpi_dma_deconfigure(struct device *dev) { } - #define ACPI_PTR(_ptr) (NULL) static inline void acpi_device_set_enumerated(struct acpi_device *adev) diff --git a/include/linux/amifd.h b/include/linux/amifd.h deleted file mode 100644 index 202a77dbe46d..000000000000 --- a/include/linux/amifd.h +++ /dev/null @@ -1,63 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _AMIFD_H -#define _AMIFD_H - -/* Definitions for the Amiga floppy driver */ - -#include <linux/fd.h> - -#define FD_MAX_UNITS 4 /* Max. Number of drives */ -#define FLOPPY_MAX_SECTORS 22 /* Max. Number of sectors per track */ - -#ifndef ASSEMBLER - -struct fd_data_type { - char *name; /* description of data type */ - int sects; /* sectors per track */ -#ifdef __STDC__ - int (*read_fkt)(int); - void (*write_fkt)(int); -#else - int (*read_fkt)(); /* read whole track */ - void (*write_fkt)(); /* write whole track */ -#endif -}; - -/* -** Floppy type descriptions -*/ - -struct fd_drive_type { - unsigned long code; /* code returned from drive */ - char *name; /* description of drive */ - unsigned int tracks; /* number of tracks */ - unsigned int heads; /* number of heads */ - unsigned int read_size; /* raw read size for one track */ - unsigned int write_size; /* raw write size for one track */ - unsigned int sect_mult; /* sectors and gap multiplier (HD = 2) */ - unsigned int precomp1; /* start track for precomp 1 */ - unsigned int precomp2; /* start track for precomp 2 */ - unsigned int step_delay; /* time (in ms) for delay after step */ - unsigned int settle_time; /* time to settle after dir change */ - unsigned int side_time; /* time needed to change sides */ -}; - -struct amiga_floppy_struct { - struct fd_drive_type *type; /* type of floppy for this unit */ - struct fd_data_type *dtype; /* type of floppy for this unit */ - int track; /* current track (-1 == unknown) */ - unsigned char *trackbuf; /* current track (kmaloc()'d */ - - int blocks; /* total # blocks on disk */ - - int changed; /* true when not known */ - int disk; /* disk in drive (-1 == unknown) */ - int motor; /* true when motor is at speed */ - int busy; /* true when drive is active */ - int dirty; /* true when trackbuf is not on disk */ - int status; /* current error code for unit */ - struct gendisk *gendisk; -}; -#endif - -#endif diff --git a/include/linux/amifdreg.h b/include/linux/amifdreg.h deleted file mode 100644 index 9b514d05ec70..000000000000 --- a/include/linux/amifdreg.h +++ /dev/null @@ -1,82 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _LINUX_AMIFDREG_H -#define _LINUX_AMIFDREG_H - -/* -** CIAAPRA bits (read only) -*/ - -#define DSKRDY (0x1<<5) /* disk ready when low */ -#define DSKTRACK0 (0x1<<4) /* head at track zero when low */ -#define DSKPROT (0x1<<3) /* disk protected when low */ -#define DSKCHANGE (0x1<<2) /* low when disk removed */ - -/* -** CIAAPRB bits (read/write) -*/ - -#define DSKMOTOR (0x1<<7) /* motor on when low */ -#define DSKSEL3 (0x1<<6) /* select drive 3 when low */ -#define DSKSEL2 (0x1<<5) /* select drive 2 when low */ -#define DSKSEL1 (0x1<<4) /* select drive 1 when low */ -#define DSKSEL0 (0x1<<3) /* select drive 0 when low */ -#define DSKSIDE (0x1<<2) /* side selection: 0 = upper, 1 = lower */ -#define DSKDIREC (0x1<<1) /* step direction: 0=in, 1=out (to trk 0) */ -#define DSKSTEP (0x1) /* pulse low to step head 1 track */ - -/* -** DSKBYTR bits (read only) -*/ - -#define DSKBYT (1<<15) /* register contains valid byte when set */ -#define DMAON (1<<14) /* disk DMA enabled */ -#define DISKWRITE (1<<13) /* disk write bit in DSKLEN enabled */ -#define WORDEQUAL (1<<12) /* DSKSYNC register match when true */ -/* bits 7-0 are data */ - -/* -** ADKCON/ADKCONR bits -*/ - -#ifndef SETCLR -#define ADK_SETCLR (1<<15) /* control bit */ -#endif -#define ADK_PRECOMP1 (1<<14) /* precompensation selection */ -#define ADK_PRECOMP0 (1<<13) /* 00=none, 01=140ns, 10=280ns, 11=500ns */ -#define ADK_MFMPREC (1<<12) /* 0=GCR precomp., 1=MFM precomp. */ -#define ADK_WORDSYNC (1<<10) /* enable DSKSYNC auto DMA */ -#define ADK_MSBSYNC (1<<9) /* when 1, enable sync on MSbit (for GCR) */ -#define ADK_FAST (1<<8) /* bit cell: 0=2us (GCR), 1=1us (MFM) */ - -/* -** DSKLEN bits -*/ - -#define DSKLEN_DMAEN (1<<15) -#define DSKLEN_WRITE (1<<14) - -/* -** INTENA/INTREQ bits -*/ - -#define DSKINDEX (0x1<<4) /* DSKINDEX bit */ - -/* -** Misc -*/ - -#define MFM_SYNC 0x4489 /* standard MFM sync value */ - -/* Values for FD_COMMAND */ -#define FD_RECALIBRATE 0x07 /* move to track 0 */ -#define FD_SEEK 0x0F /* seek track */ -#define FD_READ 0xE6 /* read with MT, MFM, SKip deleted */ -#define FD_WRITE 0xC5 /* write with MT, MFM */ -#define FD_SENSEI 0x08 /* Sense Interrupt Status */ -#define FD_SPECIFY 0x03 /* specify HUT etc */ -#define FD_FORMAT 0x4D /* format one track */ -#define FD_VERSION 0x10 /* get version code */ -#define FD_CONFIGURE 0x13 /* configure FIFO operation */ -#define FD_PERPENDICULAR 0x12 /* perpendicular r/w mode */ - -#endif /* _LINUX_AMIFDREG_H */ diff --git a/include/linux/bio.h b/include/linux/bio.h index 51371740d2a8..b47c7f716731 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -21,12 +21,8 @@ #include <linux/highmem.h> #include <linux/mempool.h> #include <linux/ioprio.h> -#include <linux/bug.h> #ifdef CONFIG_BLOCK - -#include <asm/io.h> - /* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */ #include <linux/blk_types.h> @@ -133,32 +129,6 @@ static inline bool bio_full(struct bio *bio) } /* - * will die - */ -#define bvec_to_phys(bv) (page_to_phys((bv)->bv_page) + (unsigned long) (bv)->bv_offset) - -/* - * merge helpers etc - */ - -/* Default implementation of BIOVEC_PHYS_MERGEABLE */ -#define __BIOVEC_PHYS_MERGEABLE(vec1, vec2) \ - ((bvec_to_phys((vec1)) + (vec1)->bv_len) == bvec_to_phys((vec2))) - -/* - * allow arch override, for eg virtualized architectures (put in asm/io.h) - */ -#ifndef BIOVEC_PHYS_MERGEABLE -#define BIOVEC_PHYS_MERGEABLE(vec1, vec2) \ - __BIOVEC_PHYS_MERGEABLE(vec1, vec2) -#endif - -#define __BIO_SEG_BOUNDARY(addr1, addr2, mask) \ - (((addr1) | (mask)) == (((addr2) - 1) | (mask))) -#define BIOVEC_SEG_BOUNDARY(q, b1, b2) \ - __BIO_SEG_BOUNDARY(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_boundary((q))) - -/* * drivers should _never_ use the all version - the bio may have been split * before it got to the driver and the driver won't own all of it */ @@ -170,27 +140,11 @@ static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter, { iter->bi_sector += bytes >> 9; - if (bio_no_advance_iter(bio)) { + if (bio_no_advance_iter(bio)) iter->bi_size -= bytes; - iter->bi_done += bytes; - } else { + else bvec_iter_advance(bio->bi_io_vec, iter, bytes); /* TODO: It is reasonable to complete bio with error here. */ - } -} - -static inline bool bio_rewind_iter(struct bio *bio, struct bvec_iter *iter, - unsigned int bytes) -{ - iter->bi_sector -= bytes >> 9; - - if (bio_no_advance_iter(bio)) { - iter->bi_size += bytes; - iter->bi_done -= bytes; - return true; - } - - return bvec_iter_rewind(bio->bi_io_vec, iter, bytes); } #define __bio_for_each_segment(bvl, bio, iter, start) \ @@ -353,6 +307,8 @@ struct bio_integrity_payload { unsigned short bip_max_vcnt; /* integrity bio_vec slots */ unsigned short bip_flags; /* control flags */ + struct bvec_iter bio_iter; /* for rewinding parent bio */ + struct work_struct bip_work; /* I/O completion */ struct bio_vec *bip_vec; @@ -547,23 +503,31 @@ do { \ disk_devt((bio)->bi_disk) #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) -int bio_associate_blkcg_from_page(struct bio *bio, struct page *page); +int bio_associate_blkg_from_page(struct bio *bio, struct page *page); #else -static inline int bio_associate_blkcg_from_page(struct bio *bio, - struct page *page) { return 0; } +static inline int bio_associate_blkg_from_page(struct bio *bio, + struct page *page) { return 0; } #endif #ifdef CONFIG_BLK_CGROUP -int bio_associate_blkcg(struct bio *bio, struct cgroup_subsys_state *blkcg_css); int bio_associate_blkg(struct bio *bio, struct blkcg_gq *blkg); +int bio_associate_blkg_from_css(struct bio *bio, + struct cgroup_subsys_state *css); +int bio_associate_create_blkg(struct request_queue *q, struct bio *bio); +int bio_reassociate_blkg(struct request_queue *q, struct bio *bio); void bio_disassociate_task(struct bio *bio); -void bio_clone_blkcg_association(struct bio *dst, struct bio *src); +void bio_clone_blkg_association(struct bio *dst, struct bio *src); #else /* CONFIG_BLK_CGROUP */ -static inline int bio_associate_blkcg(struct bio *bio, - struct cgroup_subsys_state *blkcg_css) { return 0; } +static inline int bio_associate_blkg_from_css(struct bio *bio, + struct cgroup_subsys_state *css) +{ return 0; } +static inline int bio_associate_create_blkg(struct request_queue *q, + struct bio *bio) { return 0; } +static inline int bio_reassociate_blkg(struct request_queue *q, struct bio *bio) +{ return 0; } static inline void bio_disassociate_task(struct bio *bio) { } -static inline void bio_clone_blkcg_association(struct bio *dst, - struct bio *src) { } +static inline void bio_clone_blkg_association(struct bio *dst, + struct bio *src) { } #endif /* CONFIG_BLK_CGROUP */ #ifdef CONFIG_HIGHMEM diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h index 6d766a19f2bb..1e76ceebeb5d 100644 --- a/include/linux/blk-cgroup.h +++ b/include/linux/blk-cgroup.h @@ -126,7 +126,7 @@ struct blkcg_gq { struct request_list rl; /* reference count */ - atomic_t refcnt; + struct percpu_ref refcnt; /* is this blkg online? protected by both blkcg and q locks */ bool online; @@ -184,6 +184,8 @@ extern struct cgroup_subsys_state * const blkcg_root_css; struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg, struct request_queue *q, bool update_hint); +struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg, + struct request_queue *q); struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, struct request_queue *q); int blkcg_init_queue(struct request_queue *q); @@ -230,22 +232,59 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, char *input, struct blkg_conf_ctx *ctx); void blkg_conf_finish(struct blkg_conf_ctx *ctx); +/** + * blkcg_css - find the current css + * + * Find the css associated with either the kthread or the current task. + * This may return a dying css, so it is up to the caller to use tryget logic + * to confirm it is alive and well. + */ +static inline struct cgroup_subsys_state *blkcg_css(void) +{ + struct cgroup_subsys_state *css; + + css = kthread_blkcg(); + if (css) + return css; + return task_css(current, io_cgrp_id); +} static inline struct blkcg *css_to_blkcg(struct cgroup_subsys_state *css) { return css ? container_of(css, struct blkcg, css) : NULL; } -static inline struct blkcg *bio_blkcg(struct bio *bio) +/** + * __bio_blkcg - internal version of bio_blkcg for bfq and cfq + * + * DO NOT USE. + * There is a flaw using this version of the function. In particular, this was + * used in a broken paradigm where association was called on the given css. It + * is possible though that the returned css from task_css() is in the process + * of dying due to migration of the current task. So it is improper to assume + * *_get() is going to succeed. Both BFQ and CFQ rely on this logic and will + * take additional work to handle more gracefully. + */ +static inline struct blkcg *__bio_blkcg(struct bio *bio) { - struct cgroup_subsys_state *css; + if (bio && bio->bi_blkg) + return bio->bi_blkg->blkcg; + return css_to_blkcg(blkcg_css()); +} - if (bio && bio->bi_css) - return css_to_blkcg(bio->bi_css); - css = kthread_blkcg(); - if (css) - return css_to_blkcg(css); - return css_to_blkcg(task_css(current, io_cgrp_id)); +/** + * bio_blkcg - grab the blkcg associated with a bio + * @bio: target bio + * + * This returns the blkcg associated with a bio, NULL if not associated. + * Callers are expected to either handle NULL or know association has been + * done prior to calling this. + */ +static inline struct blkcg *bio_blkcg(struct bio *bio) +{ + if (bio && bio->bi_blkg) + return bio->bi_blkg->blkcg; + return NULL; } static inline bool blk_cgroup_congested(void) @@ -451,26 +490,35 @@ static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen) */ static inline void blkg_get(struct blkcg_gq *blkg) { - WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0); - atomic_inc(&blkg->refcnt); + percpu_ref_get(&blkg->refcnt); } /** - * blkg_try_get - try and get a blkg reference + * blkg_tryget - try and get a blkg reference * @blkg: blkg to get * * This is for use when doing an RCU lookup of the blkg. We may be in the midst * of freeing this blkg, so we can only use it if the refcnt is not zero. */ -static inline struct blkcg_gq *blkg_try_get(struct blkcg_gq *blkg) +static inline bool blkg_tryget(struct blkcg_gq *blkg) { - if (atomic_inc_not_zero(&blkg->refcnt)) - return blkg; - return NULL; + return percpu_ref_tryget(&blkg->refcnt); } +/** + * blkg_tryget_closest - try and get a blkg ref on the closet blkg + * @blkg: blkg to get + * + * This walks up the blkg tree to find the closest non-dying blkg and returns + * the blkg that it did association with as it may not be the passed in blkg. + */ +static inline struct blkcg_gq *blkg_tryget_closest(struct blkcg_gq *blkg) +{ + while (!percpu_ref_tryget(&blkg->refcnt)) + blkg = blkg->parent; -void __blkg_release_rcu(struct rcu_head *rcu); + return blkg; +} /** * blkg_put - put a blkg reference @@ -478,9 +526,7 @@ void __blkg_release_rcu(struct rcu_head *rcu); */ static inline void blkg_put(struct blkcg_gq *blkg) { - WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0); - if (atomic_dec_and_test(&blkg->refcnt)) - call_rcu(&blkg->rcu_head, __blkg_release_rcu); + percpu_ref_put(&blkg->refcnt); } /** @@ -533,25 +579,36 @@ static inline struct request_list *blk_get_rl(struct request_queue *q, rcu_read_lock(); - blkcg = bio_blkcg(bio); + if (bio && bio->bi_blkg) { + blkcg = bio->bi_blkg->blkcg; + if (blkcg == &blkcg_root) + goto rl_use_root; + + blkg_get(bio->bi_blkg); + rcu_read_unlock(); + return &bio->bi_blkg->rl; + } - /* bypass blkg lookup and use @q->root_rl directly for root */ + blkcg = css_to_blkcg(blkcg_css()); if (blkcg == &blkcg_root) - goto root_rl; + goto rl_use_root; - /* - * Try to use blkg->rl. blkg lookup may fail under memory pressure - * or if either the blkcg or queue is going away. Fall back to - * root_rl in such cases. - */ blkg = blkg_lookup(blkcg, q); if (unlikely(!blkg)) - goto root_rl; + blkg = __blkg_lookup_create(blkcg, q); + + if (blkg->blkcg == &blkcg_root || !blkg_tryget(blkg)) + goto rl_use_root; - blkg_get(blkg); rcu_read_unlock(); return &blkg->rl; -root_rl: + + /* + * Each blkg has its own request_list, however, the root blkcg + * uses the request_queue's root_rl. This is to avoid most + * overhead for the root blkcg. + */ +rl_use_root: rcu_read_unlock(); return &q->root_rl; } @@ -797,32 +854,26 @@ static inline bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg struct bio *bio) { return false; } #endif + +static inline void blkcg_bio_issue_init(struct bio *bio) +{ + bio_issue_init(&bio->bi_issue, bio_sectors(bio)); +} + static inline bool blkcg_bio_issue_check(struct request_queue *q, struct bio *bio) { - struct blkcg *blkcg; struct blkcg_gq *blkg; bool throtl = false; rcu_read_lock(); - blkcg = bio_blkcg(bio); - - /* associate blkcg if bio hasn't attached one */ - bio_associate_blkcg(bio, &blkcg->css); - blkg = blkg_lookup(blkcg, q); - if (unlikely(!blkg)) { - spin_lock_irq(q->queue_lock); - blkg = blkg_lookup_create(blkcg, q); - if (IS_ERR(blkg)) - blkg = NULL; - spin_unlock_irq(q->queue_lock); - } + bio_associate_create_blkg(q, bio); + blkg = bio->bi_blkg; throtl = blk_throtl_bio(q, blkg, bio); if (!throtl) { - blkg = blkg ?: q->root_blkg; /* * If the bio is flagged with BIO_QUEUE_ENTERED it means this * is a split bio and we would have already accounted for the @@ -834,6 +885,8 @@ static inline bool blkcg_bio_issue_check(struct request_queue *q, blkg_rwstat_add(&blkg->stat_ios, bio->bi_opf, 1); } + blkcg_bio_issue_init(bio); + rcu_read_unlock(); return !throtl; } @@ -930,6 +983,7 @@ static inline int blkcg_activate_policy(struct request_queue *q, static inline void blkcg_deactivate_policy(struct request_queue *q, const struct blkcg_policy *pol) { } +static inline struct blkcg *__bio_blkcg(struct bio *bio) { return NULL; } static inline struct blkcg *bio_blkcg(struct bio *bio) { return NULL; } static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg, @@ -945,6 +999,7 @@ static inline void blk_put_rl(struct request_list *rl) { } static inline void blk_rq_set_rl(struct request *rq, struct request_list *rl) { } static inline struct request_list *blk_rq_rl(struct request *rq) { return &rq->q->root_rl; } +static inline void blkcg_bio_issue_init(struct bio *bio) { } static inline bool blkcg_bio_issue_check(struct request_queue *q, struct bio *bio) { return true; } diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 1da59c16f637..2286dc12c6bc 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -203,6 +203,10 @@ enum { struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *); struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, struct request_queue *q); +struct request_queue *blk_mq_init_sq_queue(struct blk_mq_tag_set *set, + const struct blk_mq_ops *ops, + unsigned int queue_depth, + unsigned int set_flags); int blk_mq_register_dev(struct device *, struct request_queue *); void blk_mq_unregister_dev(struct device *, struct request_queue *); diff --git a/include/linux/blk-pm.h b/include/linux/blk-pm.h new file mode 100644 index 000000000000..b80c65aba249 --- /dev/null +++ b/include/linux/blk-pm.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _BLK_PM_H_ +#define _BLK_PM_H_ + +struct device; +struct request_queue; + +/* + * block layer runtime pm functions + */ +#ifdef CONFIG_PM +extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev); +extern int blk_pre_runtime_suspend(struct request_queue *q); +extern void blk_post_runtime_suspend(struct request_queue *q, int err); +extern void blk_pre_runtime_resume(struct request_queue *q); +extern void blk_post_runtime_resume(struct request_queue *q, int err); +extern void blk_set_runtime_active(struct request_queue *q); +#else +static inline void blk_pm_runtime_init(struct request_queue *q, + struct device *dev) {} +#endif + +#endif /* _BLK_PM_H_ */ diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index f6dfb30737d8..9578c7ab1eb6 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -178,7 +178,6 @@ struct bio { * release. Read comment on top of bio_associate_current(). */ struct io_context *bi_ioc; - struct cgroup_subsys_state *bi_css; struct blkcg_gq *bi_blkg; struct bio_issue bi_issue; #endif diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 6980014357d4..61207560e826 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -108,7 +108,7 @@ typedef __u32 __bitwise req_flags_t; #define RQF_QUIET ((__force req_flags_t)(1 << 11)) /* elevator private data attached */ #define RQF_ELVPRIV ((__force req_flags_t)(1 << 12)) -/* account I/O stat */ +/* account into disk and partition IO statistics */ #define RQF_IO_STAT ((__force req_flags_t)(1 << 13)) /* request came from our alloc pool */ #define RQF_ALLOCED ((__force req_flags_t)(1 << 14)) @@ -116,7 +116,7 @@ typedef __u32 __bitwise req_flags_t; #define RQF_PM ((__force req_flags_t)(1 << 15)) /* on IO scheduler merge hash */ #define RQF_HASHED ((__force req_flags_t)(1 << 16)) -/* IO stats tracking on */ +/* track IO completion time */ #define RQF_STATS ((__force req_flags_t)(1 << 17)) /* Look at ->special_vec for the actual data payload instead of the bio chain. */ @@ -504,6 +504,12 @@ struct request_queue { * various queue flags, see QUEUE_* below */ unsigned long queue_flags; + /* + * Number of contexts that have called blk_set_pm_only(). If this + * counter is above zero then only RQF_PM and RQF_PREEMPT requests are + * processed. + */ + atomic_t pm_only; /* * ida allocated id for this queue. Used to index queues from @@ -679,7 +685,7 @@ struct request_queue { #define QUEUE_FLAG_FAIL_IO 7 /* fake timeout */ #define QUEUE_FLAG_NONROT 9 /* non-rotational device (SSD) */ #define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */ -#define QUEUE_FLAG_IO_STAT 10 /* do IO stats */ +#define QUEUE_FLAG_IO_STAT 10 /* do disk/partitions IO accounting */ #define QUEUE_FLAG_DISCARD 11 /* supports DISCARD */ #define QUEUE_FLAG_NOXMERGES 12 /* No extended merges */ #define QUEUE_FLAG_ADD_RANDOM 13 /* Contributes to random pool */ @@ -693,12 +699,11 @@ struct request_queue { #define QUEUE_FLAG_FUA 21 /* device supports FUA writes */ #define QUEUE_FLAG_FLUSH_NQ 22 /* flush not queueuable */ #define QUEUE_FLAG_DAX 23 /* device supports DAX */ -#define QUEUE_FLAG_STATS 24 /* track rq completion times */ +#define QUEUE_FLAG_STATS 24 /* track IO start and completion times */ #define QUEUE_FLAG_POLL_STATS 25 /* collecting stats for hybrid polling */ #define QUEUE_FLAG_REGISTERED 26 /* queue has been registered to a disk */ #define QUEUE_FLAG_SCSI_PASSTHROUGH 27 /* queue supports SCSI commands */ #define QUEUE_FLAG_QUIESCED 28 /* queue has been quiesced */ -#define QUEUE_FLAG_PREEMPT_ONLY 29 /* only process REQ_PREEMPT requests */ #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ (1 << QUEUE_FLAG_SAME_COMP) | \ @@ -736,12 +741,11 @@ bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q); ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \ REQ_FAILFAST_DRIVER)) #define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags) -#define blk_queue_preempt_only(q) \ - test_bit(QUEUE_FLAG_PREEMPT_ONLY, &(q)->queue_flags) +#define blk_queue_pm_only(q) atomic_read(&(q)->pm_only) #define blk_queue_fua(q) test_bit(QUEUE_FLAG_FUA, &(q)->queue_flags) -extern int blk_set_preempt_only(struct request_queue *q); -extern void blk_clear_preempt_only(struct request_queue *q); +extern void blk_set_pm_only(struct request_queue *q); +extern void blk_clear_pm_only(struct request_queue *q); static inline int queue_in_flight(struct request_queue *q) { @@ -1281,29 +1285,6 @@ extern void blk_put_queue(struct request_queue *); extern void blk_set_queue_dying(struct request_queue *); /* - * block layer runtime pm functions - */ -#ifdef CONFIG_PM -extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev); -extern int blk_pre_runtime_suspend(struct request_queue *q); -extern void blk_post_runtime_suspend(struct request_queue *q, int err); -extern void blk_pre_runtime_resume(struct request_queue *q); -extern void blk_post_runtime_resume(struct request_queue *q, int err); -extern void blk_set_runtime_active(struct request_queue *q); -#else -static inline void blk_pm_runtime_init(struct request_queue *q, - struct device *dev) {} -static inline int blk_pre_runtime_suspend(struct request_queue *q) -{ - return -ENOSYS; -} -static inline void blk_post_runtime_suspend(struct request_queue *q, int err) {} -static inline void blk_pre_runtime_resume(struct request_queue *q) {} -static inline void blk_post_runtime_resume(struct request_queue *q, int err) {} -static inline void blk_set_runtime_active(struct request_queue *q) {} -#endif - -/* * blk_plug permits building a queue of related requests by holding the I/O * fragments for a short period. This allows merging of sequential requests * into single larger request. As the requests are moved from a per-task list to @@ -1676,94 +1657,6 @@ static inline void put_dev_sector(Sector p) put_page(p.v); } -static inline bool __bvec_gap_to_prev(struct request_queue *q, - struct bio_vec *bprv, unsigned int offset) -{ - return offset || - ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q)); -} - -/* - * Check if adding a bio_vec after bprv with offset would create a gap in - * the SG list. Most drivers don't care about this, but some do. - */ -static inline bool bvec_gap_to_prev(struct request_queue *q, - struct bio_vec *bprv, unsigned int offset) -{ - if (!queue_virt_boundary(q)) - return false; - return __bvec_gap_to_prev(q, bprv, offset); -} - -/* - * Check if the two bvecs from two bios can be merged to one segment. - * If yes, no need to check gap between the two bios since the 1st bio - * and the 1st bvec in the 2nd bio can be handled in one segment. - */ -static inline bool bios_segs_mergeable(struct request_queue *q, - struct bio *prev, struct bio_vec *prev_last_bv, - struct bio_vec *next_first_bv) -{ - if (!BIOVEC_PHYS_MERGEABLE(prev_last_bv, next_first_bv)) - return false; - if (!BIOVEC_SEG_BOUNDARY(q, prev_last_bv, next_first_bv)) - return false; - if (prev->bi_seg_back_size + next_first_bv->bv_len > - queue_max_segment_size(q)) - return false; - return true; -} - -static inline bool bio_will_gap(struct request_queue *q, - struct request *prev_rq, - struct bio *prev, - struct bio *next) -{ - if (bio_has_data(prev) && queue_virt_boundary(q)) { - struct bio_vec pb, nb; - - /* - * don't merge if the 1st bio starts with non-zero - * offset, otherwise it is quite difficult to respect - * sg gap limit. We work hard to merge a huge number of small - * single bios in case of mkfs. - */ - if (prev_rq) - bio_get_first_bvec(prev_rq->bio, &pb); - else - bio_get_first_bvec(prev, &pb); - if (pb.bv_offset) - return true; - - /* - * We don't need to worry about the situation that the - * merged segment ends in unaligned virt boundary: - * - * - if 'pb' ends aligned, the merged segment ends aligned - * - if 'pb' ends unaligned, the next bio must include - * one single bvec of 'nb', otherwise the 'nb' can't - * merge with 'pb' - */ - bio_get_last_bvec(prev, &pb); - bio_get_first_bvec(next, &nb); - - if (!bios_segs_mergeable(q, prev, &pb, &nb)) - return __bvec_gap_to_prev(q, &pb, nb.bv_offset); - } - - return false; -} - -static inline bool req_gap_back_merge(struct request *req, struct bio *bio) -{ - return bio_will_gap(req->q, req, req->biotail, bio); -} - -static inline bool req_gap_front_merge(struct request *req, struct bio *bio) -{ - return bio_will_gap(req->q, NULL, bio, req->bio); -} - int kblockd_schedule_work(struct work_struct *work); int kblockd_schedule_work_on(int cpu, struct work_struct *work); int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay); @@ -1843,26 +1736,6 @@ queue_max_integrity_segments(struct request_queue *q) return q->limits.max_integrity_segments; } -static inline bool integrity_req_gap_back_merge(struct request *req, - struct bio *next) -{ - struct bio_integrity_payload *bip = bio_integrity(req->bio); - struct bio_integrity_payload *bip_next = bio_integrity(next); - - return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], - bip_next->bip_vec[0].bv_offset); -} - -static inline bool integrity_req_gap_front_merge(struct request *req, - struct bio *bio) -{ - struct bio_integrity_payload *bip = bio_integrity(bio); - struct bio_integrity_payload *bip_next = bio_integrity(req->bio); - - return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], - bip_next->bip_vec[0].bv_offset); -} - /** * bio_integrity_intervals - Return number of integrity intervals for a bio * @bi: blk_integrity profile for device @@ -1947,17 +1820,6 @@ static inline bool blk_integrity_merge_bio(struct request_queue *rq, return true; } -static inline bool integrity_req_gap_back_merge(struct request *req, - struct bio *next) -{ - return false; -} -static inline bool integrity_req_gap_front_merge(struct request *req, - struct bio *bio) -{ - return false; -} - static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi, unsigned int sectors) { diff --git a/include/linux/bvec.h b/include/linux/bvec.h index fe7a22dd133b..02c73c6aa805 100644 --- a/include/linux/bvec.h +++ b/include/linux/bvec.h @@ -40,8 +40,6 @@ struct bvec_iter { unsigned int bi_idx; /* current index into bvl_vec */ - unsigned int bi_done; /* number of bytes completed */ - unsigned int bi_bvec_done; /* number of bytes completed in current bvec */ }; @@ -85,7 +83,6 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv, bytes -= len; iter->bi_size -= len; iter->bi_bvec_done += len; - iter->bi_done += len; if (iter->bi_bvec_done == __bvec_iter_bvec(bv, *iter)->bv_len) { iter->bi_bvec_done = 0; diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 32c553556bbd..b8bcbdeb2eac 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -93,6 +93,8 @@ extern struct css_set init_css_set; bool css_has_online_children(struct cgroup_subsys_state *css); struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss); +struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgroup, + struct cgroup_subsys *ss); struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgroup, struct cgroup_subsys *ss); struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry, diff --git a/include/linux/compat.h b/include/linux/compat.h index 1a3c4f37e908..de0c13bdcd2c 100644 --- a/include/linux/compat.h +++ b/include/linux/compat.h @@ -103,6 +103,9 @@ typedef struct compat_sigaltstack { compat_size_t ss_size; } compat_stack_t; #endif +#ifndef COMPAT_MINSIGSTKSZ +#define COMPAT_MINSIGSTKSZ MINSIGSTKSZ +#endif #define compat_jiffies_to_clock_t(x) \ (((unsigned long)(x) * COMPAT_USER_HZ) / HZ) diff --git a/include/linux/device.h b/include/linux/device.h index 8f882549edee..983506789402 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -927,6 +927,8 @@ struct dev_links_info { * @offline: Set after successful invocation of bus type's .offline(). * @of_node_reused: Set if the device-tree node is shared with an ancestor * device. + * @dma_coherent: this particular device is dma coherent, even if the + * architecture supports non-coherent devices. * * At the lowest level, every device in a Linux system is represented by an * instance of struct device. The device structure contains the information @@ -1016,6 +1018,11 @@ struct device { bool offline_disabled:1; bool offline:1; bool of_node_reused:1; +#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ + defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ + defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) + bool dma_coherent:1; +#endif }; static inline struct device *kobj_to_dev(struct kobject *kobj) diff --git a/include/linux/dma-debug.h b/include/linux/dma-debug.h index a785f2507159..30213adbb6b9 100644 --- a/include/linux/dma-debug.h +++ b/include/linux/dma-debug.h @@ -32,6 +32,9 @@ extern void dma_debug_add_bus(struct bus_type *bus); extern int dma_debug_resize_entries(u32 num_entries); +extern void debug_dma_map_single(struct device *dev, const void *addr, + unsigned long len); + extern void debug_dma_map_page(struct device *dev, struct page *page, size_t offset, size_t size, int direction, dma_addr_t dma_addr, @@ -103,6 +106,11 @@ static inline int dma_debug_resize_entries(u32 num_entries) return 0; } +static inline void debug_dma_map_single(struct device *dev, const void *addr, + unsigned long len) +{ +} + static inline void debug_dma_map_page(struct device *dev, struct page *page, size_t offset, size_t size, int direction, dma_addr_t dma_addr, diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index 8d9f33febde5..fbca184ff5a0 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -27,7 +27,8 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) if (!dev->dma_mask) return false; - return addr + size - 1 <= *dev->dma_mask; + return addr + size - 1 <= + min_not_zero(*dev->dma_mask, dev->bus_dma_mask); } #endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */ @@ -55,10 +56,15 @@ static inline void dma_mark_clean(void *addr, size_t size) } #endif /* CONFIG_ARCH_HAS_DMA_MARK_CLEAN */ +u64 dma_direct_get_required_mask(struct device *dev); void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs); void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs); +void *dma_direct_alloc_pages(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs); +void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t dma_addr, unsigned long attrs); dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long attrs); diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 1db6a6b46d0d..15bd41447025 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -130,13 +130,10 @@ struct dma_map_ops { enum dma_data_direction direction); int (*mapping_error)(struct device *dev, dma_addr_t dma_addr); int (*dma_supported)(struct device *dev, u64 mask); -#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK u64 (*get_required_mask)(struct device *dev); -#endif }; extern const struct dma_map_ops dma_direct_ops; -extern const struct dma_map_ops dma_noncoherent_ops; extern const struct dma_map_ops dma_virt_ops; #define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1)) @@ -232,6 +229,7 @@ static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr, dma_addr_t addr; BUG_ON(!valid_dma_direction(dir)); + debug_dma_map_single(dev, ptr, size); addr = ops->map_page(dev, virt_to_page(ptr), offset_in_page(ptr), size, dir, attrs); @@ -445,7 +443,8 @@ dma_cache_sync(struct device *dev, void *vaddr, size_t size, } extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size); + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs); void *dma_common_contiguous_remap(struct page *page, size_t size, unsigned long vm_flags, @@ -477,14 +476,14 @@ dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, BUG_ON(!ops); if (ops->mmap) return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs); - return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size); + return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); } #define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0) int -dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, - void *cpu_addr, dma_addr_t dma_addr, size_t size); +dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, + dma_addr_t dma_addr, size_t size, unsigned long attrs); static inline int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr, @@ -496,7 +495,8 @@ dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr, if (ops->get_sgtable) return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs); - return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size); + return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size, + attrs); } #define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0) @@ -558,9 +558,11 @@ static inline void dma_free_attrs(struct device *dev, size_t size, } static inline void *dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag) + dma_addr_t *dma_handle, gfp_t gfp) { - return dma_alloc_attrs(dev, size, dma_handle, flag, 0); + + return dma_alloc_attrs(dev, size, dma_handle, gfp, + (gfp & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0); } static inline void dma_free_coherent(struct device *dev, size_t size, @@ -753,18 +755,6 @@ dma_mark_declared_memory_occupied(struct device *dev, } #endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */ -#ifdef CONFIG_HAS_DMA -int dma_configure(struct device *dev); -void dma_deconfigure(struct device *dev); -#else -static inline int dma_configure(struct device *dev) -{ - return 0; -} - -static inline void dma_deconfigure(struct device *dev) {} -#endif - /* * Managed DMA API */ @@ -806,8 +796,12 @@ static inline void dmam_release_declared_memory(struct device *dev) static inline void *dma_alloc_wc(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t gfp) { - return dma_alloc_attrs(dev, size, dma_addr, gfp, - DMA_ATTR_WRITE_COMBINE); + unsigned long attrs = DMA_ATTR_NO_WARN; + + if (gfp & __GFP_NOWARN) + attrs |= DMA_ATTR_NO_WARN; + + return dma_alloc_attrs(dev, size, dma_addr, gfp, attrs); } #ifndef dma_alloc_writecombine #define dma_alloc_writecombine dma_alloc_wc diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h index a0aa00cc909d..9051b055beec 100644 --- a/include/linux/dma-noncoherent.h +++ b/include/linux/dma-noncoherent.h @@ -4,18 +4,35 @@ #include <linux/dma-mapping.h> +#ifdef CONFIG_ARCH_HAS_DMA_COHERENCE_H +#include <asm/dma-coherence.h> +#elif defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ + defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ + defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) +static inline bool dev_is_dma_coherent(struct device *dev) +{ + return dev->dma_coherent; +} +#else +static inline bool dev_is_dma_coherent(struct device *dev) +{ + return true; +} +#endif /* CONFIG_ARCH_HAS_DMA_COHERENCE_H */ + void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs); void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs); +long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr, + dma_addr_t dma_addr); -#ifdef CONFIG_DMA_NONCOHERENT_MMAP -int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, +#ifdef CONFIG_ARCH_HAS_DMA_MMAP_PGPROT +pgprot_t arch_dma_mmap_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs); #else -#define arch_dma_mmap NULL -#endif /* CONFIG_DMA_NONCOHERENT_MMAP */ +# define arch_dma_mmap_pgprot(dev, prot, attrs) pgprot_noncached(prot) +#endif #ifdef CONFIG_DMA_NONCOHERENT_CACHE_SYNC void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size, diff --git a/include/linux/elevator.h b/include/linux/elevator.h index a02deea30185..015bb59c0331 100644 --- a/include/linux/elevator.h +++ b/include/linux/elevator.h @@ -111,7 +111,7 @@ struct elevator_mq_ops { void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool); struct request *(*dispatch_request)(struct blk_mq_hw_ctx *); bool (*has_work)(struct blk_mq_hw_ctx *); - void (*completed_request)(struct request *); + void (*completed_request)(struct request *, u64); void (*started_request)(struct request *); void (*requeue_request)(struct request *); struct request *(*former_request)(struct request_queue *, struct request *); diff --git a/include/linux/fsl_ifc.h b/include/linux/fsl_ifc.h index 3fdfede2f0f3..5f343b796ad9 100644 --- a/include/linux/fsl_ifc.h +++ b/include/linux/fsl_ifc.h @@ -274,6 +274,8 @@ */ /* Auto Boot Mode */ #define IFC_NAND_NCFGR_BOOT 0x80000000 +/* SRAM Initialization */ +#define IFC_NAND_NCFGR_SRAM_INIT_EN 0x20000000 /* Addressing Mode-ROW0+n/COL0 */ #define IFC_NAND_NCFGR_ADDR_MODE_RC0 0x00000000 /* Addressing Mode-ROW0+n/COL0+n */ diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 25c08c6c7f99..70fc838e6773 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -402,10 +402,11 @@ static inline void free_part_info(struct hd_struct *part) extern void part_round_stats(struct request_queue *q, int cpu, struct hd_struct *part); /* block/genhd.c */ -extern void device_add_disk(struct device *parent, struct gendisk *disk); +extern void device_add_disk(struct device *parent, struct gendisk *disk, + const struct attribute_group **groups); static inline void add_disk(struct gendisk *disk) { - device_add_disk(NULL, disk); + device_add_disk(NULL, disk, NULL); } extern void device_add_disk_no_queue_reg(struct device *parent, struct gendisk *disk); static inline void add_disk_no_queue_reg(struct gendisk *disk) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 99c19b06d9a4..fdcb45999b26 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -43,7 +43,7 @@ extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, unsigned char *vec); extern bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, unsigned long new_addr, unsigned long old_end, - pmd_t *old_pmd, pmd_t *new_pmd, bool *need_flush); + pmd_t *old_pmd, pmd_t *new_pmd); extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, pgprot_t newprot, int prot_numa); diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h index 9493d4a388db..99e0c1b0b5fb 100644 --- a/include/linux/hwmon.h +++ b/include/linux/hwmon.h @@ -118,6 +118,7 @@ enum hwmon_in_attributes { hwmon_in_max_alarm, hwmon_in_lcrit_alarm, hwmon_in_crit_alarm, + hwmon_in_enable, }; #define HWMON_I_INPUT BIT(hwmon_in_input) @@ -135,6 +136,7 @@ enum hwmon_in_attributes { #define HWMON_I_MAX_ALARM BIT(hwmon_in_max_alarm) #define HWMON_I_LCRIT_ALARM BIT(hwmon_in_lcrit_alarm) #define HWMON_I_CRIT_ALARM BIT(hwmon_in_crit_alarm) +#define HWMON_I_ENABLE BIT(hwmon_in_enable) enum hwmon_curr_attributes { hwmon_curr_input, diff --git a/include/linux/lightnvm.h b/include/linux/lightnvm.h index e9e0d1c7eaf5..2fdeac1a420d 100644 --- a/include/linux/lightnvm.h +++ b/include/linux/lightnvm.h @@ -86,8 +86,8 @@ struct nvm_chk_meta; typedef int (nvm_id_fn)(struct nvm_dev *); typedef int (nvm_op_bb_tbl_fn)(struct nvm_dev *, struct ppa_addr, u8 *); typedef int (nvm_op_set_bb_fn)(struct nvm_dev *, struct ppa_addr *, int, int); -typedef int (nvm_get_chk_meta_fn)(struct nvm_dev *, struct nvm_chk_meta *, - sector_t, int); +typedef int (nvm_get_chk_meta_fn)(struct nvm_dev *, sector_t, int, + struct nvm_chk_meta *); typedef int (nvm_submit_io_fn)(struct nvm_dev *, struct nvm_rq *); typedef int (nvm_submit_io_sync_fn)(struct nvm_dev *, struct nvm_rq *); typedef void *(nvm_create_dma_pool_fn)(struct nvm_dev *, char *); @@ -305,6 +305,8 @@ struct nvm_rq { u64 ppa_status; /* ppa media status */ int error; + int is_seq; /* Sequential hint flag. 1.2 only */ + void *private; }; @@ -318,6 +320,11 @@ static inline void *nvm_rq_to_pdu(struct nvm_rq *rqdata) return rqdata + 1; } +static inline struct ppa_addr *nvm_rq_to_ppa_list(struct nvm_rq *rqd) +{ + return (rqd->nr_ppas > 1) ? rqd->ppa_list : &rqd->ppa_addr; +} + enum { NVM_BLK_ST_FREE = 0x1, /* Free block */ NVM_BLK_ST_TGT = 0x2, /* Block in use by target */ @@ -485,6 +492,144 @@ static inline struct ppa_addr dev_to_generic_addr(struct nvm_dev *dev, return l; } +static inline u64 dev_to_chunk_addr(struct nvm_dev *dev, void *addrf, + struct ppa_addr p) +{ + struct nvm_geo *geo = &dev->geo; + u64 caddr; + + if (geo->version == NVM_OCSSD_SPEC_12) { + struct nvm_addrf_12 *ppaf = (struct nvm_addrf_12 *)addrf; + + caddr = (u64)p.g.pg << ppaf->pg_offset; + caddr |= (u64)p.g.pl << ppaf->pln_offset; + caddr |= (u64)p.g.sec << ppaf->sec_offset; + } else { + caddr = p.m.sec; + } + + return caddr; +} + +static inline struct ppa_addr nvm_ppa32_to_ppa64(struct nvm_dev *dev, + void *addrf, u32 ppa32) +{ + struct ppa_addr ppa64; + + ppa64.ppa = 0; + + if (ppa32 == -1) { + ppa64.ppa = ADDR_EMPTY; + } else if (ppa32 & (1U << 31)) { + ppa64.c.line = ppa32 & ((~0U) >> 1); + ppa64.c.is_cached = 1; + } else { + struct nvm_geo *geo = &dev->geo; + + if (geo->version == NVM_OCSSD_SPEC_12) { + struct nvm_addrf_12 *ppaf = addrf; + + ppa64.g.ch = (ppa32 & ppaf->ch_mask) >> + ppaf->ch_offset; + ppa64.g.lun = (ppa32 & ppaf->lun_mask) >> + ppaf->lun_offset; + ppa64.g.blk = (ppa32 & ppaf->blk_mask) >> + ppaf->blk_offset; + ppa64.g.pg = (ppa32 & ppaf->pg_mask) >> + ppaf->pg_offset; + ppa64.g.pl = (ppa32 & ppaf->pln_mask) >> + ppaf->pln_offset; + ppa64.g.sec = (ppa32 & ppaf->sec_mask) >> + ppaf->sec_offset; + } else { + struct nvm_addrf *lbaf = addrf; + + ppa64.m.grp = (ppa32 & lbaf->ch_mask) >> + lbaf->ch_offset; + ppa64.m.pu = (ppa32 & lbaf->lun_mask) >> + lbaf->lun_offset; + ppa64.m.chk = (ppa32 & lbaf->chk_mask) >> + lbaf->chk_offset; + ppa64.m.sec = (ppa32 & lbaf->sec_mask) >> + lbaf->sec_offset; + } + } + + return ppa64; +} + +static inline u32 nvm_ppa64_to_ppa32(struct nvm_dev *dev, + void *addrf, struct ppa_addr ppa64) +{ + u32 ppa32 = 0; + + if (ppa64.ppa == ADDR_EMPTY) { + ppa32 = ~0U; + } else if (ppa64.c.is_cached) { + ppa32 |= ppa64.c.line; + ppa32 |= 1U << 31; + } else { + struct nvm_geo *geo = &dev->geo; + + if (geo->version == NVM_OCSSD_SPEC_12) { + struct nvm_addrf_12 *ppaf = addrf; + + ppa32 |= ppa64.g.ch << ppaf->ch_offset; + ppa32 |= ppa64.g.lun << ppaf->lun_offset; + ppa32 |= ppa64.g.blk << ppaf->blk_offset; + ppa32 |= ppa64.g.pg << ppaf->pg_offset; + ppa32 |= ppa64.g.pl << ppaf->pln_offset; + ppa32 |= ppa64.g.sec << ppaf->sec_offset; + } else { + struct nvm_addrf *lbaf = addrf; + + ppa32 |= ppa64.m.grp << lbaf->ch_offset; + ppa32 |= ppa64.m.pu << lbaf->lun_offset; + ppa32 |= ppa64.m.chk << lbaf->chk_offset; + ppa32 |= ppa64.m.sec << lbaf->sec_offset; + } + } + + return ppa32; +} + +static inline int nvm_next_ppa_in_chk(struct nvm_tgt_dev *dev, + struct ppa_addr *ppa) +{ + struct nvm_geo *geo = &dev->geo; + int last = 0; + + if (geo->version == NVM_OCSSD_SPEC_12) { + int sec = ppa->g.sec; + + sec++; + if (sec == geo->ws_min) { + int pg = ppa->g.pg; + + sec = 0; + pg++; + if (pg == geo->num_pg) { + int pl = ppa->g.pl; + + pg = 0; + pl++; + if (pl == geo->num_pln) + last = 1; + + ppa->g.pl = pl; + } + ppa->g.pg = pg; + } + ppa->g.sec = sec; + } else { + ppa->m.sec++; + if (ppa->m.sec == geo->clba) + last = 1; + } + + return last; +} + typedef blk_qc_t (nvm_tgt_make_rq_fn)(struct request_queue *, struct bio *); typedef sector_t (nvm_tgt_capacity_fn)(void *); typedef void *(nvm_tgt_init_fn)(struct nvm_tgt_dev *, struct gendisk *, @@ -493,9 +638,15 @@ typedef void (nvm_tgt_exit_fn)(void *, bool); typedef int (nvm_tgt_sysfs_init_fn)(struct gendisk *); typedef void (nvm_tgt_sysfs_exit_fn)(struct gendisk *); +enum { + NVM_TGT_F_DEV_L2P = 0, + NVM_TGT_F_HOST_L2P = 1 << 0, +}; + struct nvm_tgt_type { const char *name; unsigned int version[3]; + int flags; /* target entry points */ nvm_tgt_make_rq_fn *make_rq; @@ -524,18 +675,13 @@ extern struct nvm_dev *nvm_alloc_dev(int); extern int nvm_register(struct nvm_dev *); extern void nvm_unregister(struct nvm_dev *); - -extern int nvm_get_chunk_meta(struct nvm_tgt_dev *tgt_dev, - struct nvm_chk_meta *meta, struct ppa_addr ppa, - int nchks); - -extern int nvm_set_tgt_bb_tbl(struct nvm_tgt_dev *, struct ppa_addr *, +extern int nvm_get_chunk_meta(struct nvm_tgt_dev *, struct ppa_addr, + int, struct nvm_chk_meta *); +extern int nvm_set_chunk_meta(struct nvm_tgt_dev *, struct ppa_addr *, int, int); extern int nvm_submit_io(struct nvm_tgt_dev *, struct nvm_rq *); extern int nvm_submit_io_sync(struct nvm_tgt_dev *, struct nvm_rq *); extern void nvm_end_io(struct nvm_rq *); -extern int nvm_bb_tbl_fold(struct nvm_dev *, u8 *, int); -extern int nvm_get_tgt_bb_tbl(struct nvm_tgt_dev *, struct ppa_addr, u8 *); #else /* CONFIG_NVM */ struct nvm_dev_ops; diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 66d94b4557cf..88a041b73abf 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -1032,6 +1032,14 @@ static inline void *mlx5_frag_buf_get_wqe(struct mlx5_frag_buf_ctrl *fbc, ((fbc->frag_sz_m1 & ix) << fbc->log_stride); } +static inline u32 +mlx5_frag_buf_get_idx_last_contig_stride(struct mlx5_frag_buf_ctrl *fbc, u32 ix) +{ + u32 last_frag_stride_idx = (ix + fbc->strides_offset) | fbc->frag_sz_m1; + + return min_t(u32, last_frag_stride_idx - fbc->strides_offset, fbc->sz_m1); +} + int mlx5_cmd_init(struct mlx5_core_dev *dev); void mlx5_cmd_cleanup(struct mlx5_core_dev *dev); void mlx5_cmd_use_events(struct mlx5_core_dev *dev); diff --git a/include/linux/module.h b/include/linux/module.h index f807f15bebbe..e19ae08c7fb8 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -20,6 +20,7 @@ #include <linux/export.h> #include <linux/rbtree_latch.h> #include <linux/error-injection.h> +#include <linux/tracepoint-defs.h> #include <linux/percpu.h> #include <asm/module.h> @@ -430,7 +431,7 @@ struct module { #ifdef CONFIG_TRACEPOINTS unsigned int num_tracepoints; - struct tracepoint * const *tracepoints_ptrs; + tracepoint_ptr_t *tracepoints_ptrs; #endif #ifdef HAVE_JUMP_LABEL struct jump_entry *jump_entries; diff --git a/include/linux/mtd/blktrans.h b/include/linux/mtd/blktrans.h index e93837f647de..1d3ade69d39a 100644 --- a/include/linux/mtd/blktrans.h +++ b/include/linux/mtd/blktrans.h @@ -23,7 +23,6 @@ #include <linux/mutex.h> #include <linux/kref.h> #include <linux/sysfs.h> -#include <linux/workqueue.h> struct hd_geometry; struct mtd_info; @@ -44,9 +43,9 @@ struct mtd_blktrans_dev { struct kref ref; struct gendisk *disk; struct attribute_group *disk_attributes; - struct workqueue_struct *wq; - struct work_struct work; struct request_queue *rq; + struct list_head rq_list; + struct blk_mq_tag_set *tag_set; spinlock_t queue_lock; void *priv; fmode_t file_mode; diff --git a/include/linux/mtd/jedec.h b/include/linux/mtd/jedec.h new file mode 100644 index 000000000000..0b6b59f7cfbd --- /dev/null +++ b/include/linux/mtd/jedec.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright © 2000-2010 David Woodhouse <dwmw2@infradead.org> + * Steven J. Hill <sjhill@realitydiluted.com> + * Thomas Gleixner <tglx@linutronix.de> + * + * Contains all JEDEC related definitions + */ + +#ifndef __LINUX_MTD_JEDEC_H +#define __LINUX_MTD_JEDEC_H + +struct jedec_ecc_info { + u8 ecc_bits; + u8 codeword_size; + __le16 bb_per_lun; + __le16 block_endurance; + u8 reserved[2]; +} __packed; + +/* JEDEC features */ +#define JEDEC_FEATURE_16_BIT_BUS (1 << 0) + +struct nand_jedec_params { + /* rev info and features block */ + /* 'J' 'E' 'S' 'D' */ + u8 sig[4]; + __le16 revision; + __le16 features; + u8 opt_cmd[3]; + __le16 sec_cmd; + u8 num_of_param_pages; + u8 reserved0[18]; + + /* manufacturer information block */ + char manufacturer[12]; + char model[20]; + u8 jedec_id[6]; + u8 reserved1[10]; + + /* memory organization block */ + __le32 byte_per_page; + __le16 spare_bytes_per_page; + u8 reserved2[6]; + __le32 pages_per_block; + __le32 blocks_per_lun; + u8 lun_count; + u8 addr_cycles; + u8 bits_per_cell; + u8 programs_per_page; + u8 multi_plane_addr; + u8 multi_plane_op_attr; + u8 reserved3[38]; + + /* electrical parameter block */ + __le16 async_sdr_speed_grade; + __le16 toggle_ddr_speed_grade; + __le16 sync_ddr_speed_grade; + u8 async_sdr_features; + u8 toggle_ddr_features; + u8 sync_ddr_features; + __le16 t_prog; + __le16 t_bers; + __le16 t_r; + __le16 t_r_multi_plane; + __le16 t_ccs; + __le16 io_pin_capacitance_typ; + __le16 input_pin_capacitance_typ; + __le16 clk_pin_capacitance_typ; + u8 driver_strength_support; + __le16 t_adl; + u8 reserved4[36]; + + /* ECC and endurance block */ + u8 guaranteed_good_blocks; + __le16 guaranteed_block_endurance; + struct jedec_ecc_info ecc_info[4]; + u8 reserved5[29]; + + /* reserved */ + u8 reserved6[148]; + + /* vendor */ + __le16 vendor_rev_num; + u8 reserved7[88]; + + /* CRC for Parameter Page */ + __le16 crc; +} __packed; + +#endif /* __LINUX_MTD_JEDEC_H */ diff --git a/include/linux/mtd/nand_bch.h b/include/linux/mtd/nand_bch.h index 98f20ef05d60..b8106651f807 100644 --- a/include/linux/mtd/nand_bch.h +++ b/include/linux/mtd/nand_bch.h @@ -12,6 +12,7 @@ #define __MTD_NAND_BCH_H__ struct mtd_info; +struct nand_chip; struct nand_bch_control; #if defined(CONFIG_MTD_NAND_ECC_BCH) @@ -21,14 +22,14 @@ static inline int mtd_nand_has_bch(void) { return 1; } /* * Calculate BCH ecc code */ -int nand_bch_calculate_ecc(struct mtd_info *mtd, const u_char *dat, +int nand_bch_calculate_ecc(struct nand_chip *chip, const u_char *dat, u_char *ecc_code); /* * Detect and correct bit errors */ -int nand_bch_correct_data(struct mtd_info *mtd, u_char *dat, u_char *read_ecc, - u_char *calc_ecc); +int nand_bch_correct_data(struct nand_chip *chip, u_char *dat, + u_char *read_ecc, u_char *calc_ecc); /* * Initialize BCH encoder/decoder */ @@ -43,14 +44,14 @@ void nand_bch_free(struct nand_bch_control *nbc); static inline int mtd_nand_has_bch(void) { return 0; } static inline int -nand_bch_calculate_ecc(struct mtd_info *mtd, const u_char *dat, +nand_bch_calculate_ecc(struct nand_chip *chip, const u_char *dat, u_char *ecc_code) { return -1; } static inline int -nand_bch_correct_data(struct mtd_info *mtd, unsigned char *buf, +nand_bch_correct_data(struct nand_chip *chip, unsigned char *buf, unsigned char *read_ecc, unsigned char *calc_ecc) { return -ENOTSUPP; diff --git a/include/linux/mtd/nand_ecc.h b/include/linux/mtd/nand_ecc.h index 8a2decf7462c..0b3bb156c344 100644 --- a/include/linux/mtd/nand_ecc.h +++ b/include/linux/mtd/nand_ecc.h @@ -13,28 +13,30 @@ #ifndef __MTD_NAND_ECC_H__ #define __MTD_NAND_ECC_H__ -struct mtd_info; +struct nand_chip; /* * Calculate 3 byte ECC code for eccsize byte block */ void __nand_calculate_ecc(const u_char *dat, unsigned int eccsize, - u_char *ecc_code); + u_char *ecc_code, bool sm_order); /* * Calculate 3 byte ECC code for 256/512 byte block */ -int nand_calculate_ecc(struct mtd_info *mtd, const u_char *dat, u_char *ecc_code); +int nand_calculate_ecc(struct nand_chip *chip, const u_char *dat, + u_char *ecc_code); /* * Detect and correct a 1 bit error for eccsize byte block */ int __nand_correct_data(u_char *dat, u_char *read_ecc, u_char *calc_ecc, - unsigned int eccsize); + unsigned int eccsize, bool sm_order); /* * Detect and correct a 1 bit error for 256/512 byte block */ -int nand_correct_data(struct mtd_info *mtd, u_char *dat, u_char *read_ecc, u_char *calc_ecc); +int nand_correct_data(struct nand_chip *chip, u_char *dat, u_char *read_ecc, + u_char *calc_ecc); #endif /* __MTD_NAND_ECC_H__ */ diff --git a/include/linux/mtd/onfi.h b/include/linux/mtd/onfi.h new file mode 100644 index 000000000000..339ac798568e --- /dev/null +++ b/include/linux/mtd/onfi.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright © 2000-2010 David Woodhouse <dwmw2@infradead.org> + * Steven J. Hill <sjhill@realitydiluted.com> + * Thomas Gleixner <tglx@linutronix.de> + * + * Contains all ONFI related definitions + */ + +#ifndef __LINUX_MTD_ONFI_H +#define __LINUX_MTD_ONFI_H + +#include <linux/types.h> + +/* ONFI version bits */ +#define ONFI_VERSION_1_0 BIT(1) +#define ONFI_VERSION_2_0 BIT(2) +#define ONFI_VERSION_2_1 BIT(3) +#define ONFI_VERSION_2_2 BIT(4) +#define ONFI_VERSION_2_3 BIT(5) +#define ONFI_VERSION_3_0 BIT(6) +#define ONFI_VERSION_3_1 BIT(7) +#define ONFI_VERSION_3_2 BIT(8) +#define ONFI_VERSION_4_0 BIT(9) + +/* ONFI features */ +#define ONFI_FEATURE_16_BIT_BUS (1 << 0) +#define ONFI_FEATURE_EXT_PARAM_PAGE (1 << 7) + +/* ONFI timing mode, used in both asynchronous and synchronous mode */ +#define ONFI_TIMING_MODE_0 (1 << 0) +#define ONFI_TIMING_MODE_1 (1 << 1) +#define ONFI_TIMING_MODE_2 (1 << 2) +#define ONFI_TIMING_MODE_3 (1 << 3) +#define ONFI_TIMING_MODE_4 (1 << 4) +#define ONFI_TIMING_MODE_5 (1 << 5) +#define ONFI_TIMING_MODE_UNKNOWN (1 << 6) + +/* ONFI feature number/address */ +#define ONFI_FEATURE_NUMBER 256 +#define ONFI_FEATURE_ADDR_TIMING_MODE 0x1 + +/* Vendor-specific feature address (Micron) */ +#define ONFI_FEATURE_ADDR_READ_RETRY 0x89 +#define ONFI_FEATURE_ON_DIE_ECC 0x90 +#define ONFI_FEATURE_ON_DIE_ECC_EN BIT(3) + +/* ONFI subfeature parameters length */ +#define ONFI_SUBFEATURE_PARAM_LEN 4 + +/* ONFI optional commands SET/GET FEATURES supported? */ +#define ONFI_OPT_CMD_SET_GET_FEATURES (1 << 2) + +struct nand_onfi_params { + /* rev info and features block */ + /* 'O' 'N' 'F' 'I' */ + u8 sig[4]; + __le16 revision; + __le16 features; + __le16 opt_cmd; + u8 reserved0[2]; + __le16 ext_param_page_length; /* since ONFI 2.1 */ + u8 num_of_param_pages; /* since ONFI 2.1 */ + u8 reserved1[17]; + + /* manufacturer information block */ + char manufacturer[12]; + char model[20]; + u8 jedec_id; + __le16 date_code; + u8 reserved2[13]; + + /* memory organization block */ + __le32 byte_per_page; + __le16 spare_bytes_per_page; + __le32 data_bytes_per_ppage; + __le16 spare_bytes_per_ppage; + __le32 pages_per_block; + __le32 blocks_per_lun; + u8 lun_count; + u8 addr_cycles; + u8 bits_per_cell; + __le16 bb_per_lun; + __le16 block_endurance; + u8 guaranteed_good_blocks; + __le16 guaranteed_block_endurance; + u8 programs_per_page; + u8 ppage_attr; + u8 ecc_bits; + u8 interleaved_bits; + u8 interleaved_ops; + u8 reserved3[13]; + + /* electrical parameter block */ + u8 io_pin_capacitance_max; + __le16 async_timing_mode; + __le16 program_cache_timing_mode; + __le16 t_prog; + __le16 t_bers; + __le16 t_r; + __le16 t_ccs; + __le16 src_sync_timing_mode; + u8 src_ssync_features; + __le16 clk_pin_capacitance_typ; + __le16 io_pin_capacitance_typ; + __le16 input_pin_capacitance_typ; + u8 input_pin_capacitance_max; + u8 driver_strength_support; + __le16 t_int_r; + __le16 t_adl; + u8 reserved4[8]; + + /* vendor */ + __le16 vendor_revision; + u8 vendor[88]; + + __le16 crc; +} __packed; + +#define ONFI_CRC_BASE 0x4F4E + +/* Extended ECC information Block Definition (since ONFI 2.1) */ +struct onfi_ext_ecc_info { + u8 ecc_bits; + u8 codeword_size; + __le16 bb_per_lun; + __le16 block_endurance; + u8 reserved[2]; +} __packed; + +#define ONFI_SECTION_TYPE_0 0 /* Unused section. */ +#define ONFI_SECTION_TYPE_1 1 /* for additional sections. */ +#define ONFI_SECTION_TYPE_2 2 /* for ECC information. */ +struct onfi_ext_section { + u8 type; + u8 length; +} __packed; + +#define ONFI_EXT_SECTION_MAX 8 + +/* Extended Parameter Page Definition (since ONFI 2.1) */ +struct onfi_ext_param_page { + __le16 crc; + u8 sig[4]; /* 'E' 'P' 'P' 'S' */ + u8 reserved0[10]; + struct onfi_ext_section sections[ONFI_EXT_SECTION_MAX]; + + /* + * The actual size of the Extended Parameter Page is in + * @ext_param_page_length of nand_onfi_params{}. + * The following are the variable length sections. + * So we do not add any fields below. Please see the ONFI spec. + */ +} __packed; + +/** + * struct onfi_params - ONFI specific parameters that will be reused + * @version: ONFI version (BCD encoded), 0 if ONFI is not supported + * @tPROG: Page program time + * @tBERS: Block erase time + * @tR: Page read time + * @tCCS: Change column setup time + * @async_timing_mode: Supported asynchronous timing mode + * @vendor_revision: Vendor specific revision number + * @vendor: Vendor specific data + */ +struct onfi_params { + int version; + u16 tPROG; + u16 tBERS; + u16 tR; + u16 tCCS; + u16 async_timing_mode; + u16 vendor_revision; + u8 vendor[88]; +}; + +#endif /* __LINUX_MTD_ONFI_H */ diff --git a/include/linux/mtd/platnand.h b/include/linux/mtd/platnand.h new file mode 100644 index 000000000000..bc11eb6b593b --- /dev/null +++ b/include/linux/mtd/platnand.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright © 2000-2010 David Woodhouse <dwmw2@infradead.org> + * Steven J. Hill <sjhill@realitydiluted.com> + * Thomas Gleixner <tglx@linutronix.de> + * + * Contains all platform NAND related definitions. + */ + +#ifndef __LINUX_MTD_PLATNAND_H +#define __LINUX_MTD_PLATNAND_H + +#include <linux/mtd/partitions.h> +#include <linux/mtd/rawnand.h> +#include <linux/platform_device.h> + +/** + * struct platform_nand_chip - chip level device structure + * @nr_chips: max. number of chips to scan for + * @chip_offset: chip number offset + * @nr_partitions: number of partitions pointed to by partitions (or zero) + * @partitions: mtd partition list + * @chip_delay: R/B delay value in us + * @options: Option flags, e.g. 16bit buswidth + * @bbt_options: BBT option flags, e.g. NAND_BBT_USE_FLASH + * @part_probe_types: NULL-terminated array of probe types + */ +struct platform_nand_chip { + int nr_chips; + int chip_offset; + int nr_partitions; + struct mtd_partition *partitions; + int chip_delay; + unsigned int options; + unsigned int bbt_options; + const char **part_probe_types; +}; + +/** + * struct platform_nand_ctrl - controller level device structure + * @probe: platform specific function to probe/setup hardware + * @remove: platform specific function to remove/teardown hardware + * @dev_ready: platform specific function to read ready/busy pin + * @select_chip: platform specific chip select function + * @cmd_ctrl: platform specific function for controlling + * ALE/CLE/nCE. Also used to write command and address + * @write_buf: platform specific function for write buffer + * @read_buf: platform specific function for read buffer + * @priv: private data to transport driver specific settings + * + * All fields are optional and depend on the hardware driver requirements + */ +struct platform_nand_ctrl { + int (*probe)(struct platform_device *pdev); + void (*remove)(struct platform_device *pdev); + int (*dev_ready)(struct nand_chip *chip); + void (*select_chip)(struct nand_chip *chip, int cs); + void (*cmd_ctrl)(struct nand_chip *chip, int dat, unsigned int ctrl); + void (*write_buf)(struct nand_chip *chip, const uint8_t *buf, int len); + void (*read_buf)(struct nand_chip *chip, uint8_t *buf, int len); + void *priv; +}; + +/** + * struct platform_nand_data - container structure for platform-specific data + * @chip: chip level chip structure + * @ctrl: controller level device structure + */ +struct platform_nand_data { + struct platform_nand_chip chip; + struct platform_nand_ctrl ctrl; +}; + +#endif /* __LINUX_MTD_PLATNAND_H */ diff --git a/include/linux/mtd/rawnand.h b/include/linux/mtd/rawnand.h index efb2345359bb..e10b126e148f 100644 --- a/include/linux/mtd/rawnand.h +++ b/include/linux/mtd/rawnand.h @@ -21,22 +21,12 @@ #include <linux/mtd/mtd.h> #include <linux/mtd/flashchip.h> #include <linux/mtd/bbm.h> +#include <linux/mtd/jedec.h> +#include <linux/mtd/onfi.h> #include <linux/of.h> #include <linux/types.h> -struct nand_flash_dev; - -/* Scan and identify a NAND device */ -int nand_scan_with_ids(struct mtd_info *mtd, int max_chips, - struct nand_flash_dev *ids); - -static inline int nand_scan(struct mtd_info *mtd, int max_chips) -{ - return nand_scan_with_ids(mtd, max_chips, NULL); -} - -/* Internal helper for board drivers which need to override command function */ -void nand_wait_ready(struct mtd_info *mtd); +struct nand_chip; /* The maximum number of NAND chips in an array */ #define NAND_MAX_CHIPS 8 @@ -131,9 +121,11 @@ enum nand_ecc_algo { #define NAND_ECC_GENERIC_ERASED_CHECK BIT(0) #define NAND_ECC_MAXIMIZE BIT(1) -/* Bit mask for flags passed to do_nand_read_ecc */ -#define NAND_GET_DEVICE 0x80 - +/* + * When using software implementation of Hamming, we can specify which byte + * ordering should be used. + */ +#define NAND_ECC_SOFT_HAMMING_SM_ORDER BIT(2) /* * Option constants for bizarre disfunctionality and real @@ -175,9 +167,7 @@ enum nand_ecc_algo { #define NAND_SAMSUNG_LP_OPTIONS NAND_CACHEPRG /* Macros to identify the above */ -#define NAND_HAS_CACHEPROG(chip) ((chip->options & NAND_CACHEPRG)) #define NAND_HAS_SUBPAGE_READ(chip) ((chip->options & NAND_SUBPAGE_READ)) -#define NAND_HAS_SUBPAGE_WRITE(chip) !((chip)->options & NAND_NO_SUBPAGE_WRITE) /* Non chip related options */ /* This option skips the bbt scan during initialization. */ @@ -198,10 +188,10 @@ enum nand_ecc_algo { #define NAND_USE_BOUNCE_BUFFER 0x00100000 /* - * In case your controller is implementing ->cmd_ctrl() and is relying on the - * default ->cmdfunc() implementation, you may want to let the core handle the - * tCCS delay which is required when a column change (RNDIN or RNDOUT) is - * requested. + * In case your controller is implementing ->legacy.cmd_ctrl() and is relying + * on the default ->cmdfunc() implementation, you may want to let the core + * handle the tCCS delay which is required when a column change (RNDIN or + * RNDOUT) is requested. * If your controller already takes care of this delay, you don't need to set * this flag. */ @@ -222,250 +212,6 @@ enum nand_ecc_algo { #define NAND_CI_CELLTYPE_MSK 0x0C #define NAND_CI_CELLTYPE_SHIFT 2 -/* Keep gcc happy */ -struct nand_chip; - -/* ONFI version bits */ -#define ONFI_VERSION_1_0 BIT(1) -#define ONFI_VERSION_2_0 BIT(2) -#define ONFI_VERSION_2_1 BIT(3) -#define ONFI_VERSION_2_2 BIT(4) -#define ONFI_VERSION_2_3 BIT(5) -#define ONFI_VERSION_3_0 BIT(6) -#define ONFI_VERSION_3_1 BIT(7) -#define ONFI_VERSION_3_2 BIT(8) -#define ONFI_VERSION_4_0 BIT(9) - -/* ONFI features */ -#define ONFI_FEATURE_16_BIT_BUS (1 << 0) -#define ONFI_FEATURE_EXT_PARAM_PAGE (1 << 7) - -/* ONFI timing mode, used in both asynchronous and synchronous mode */ -#define ONFI_TIMING_MODE_0 (1 << 0) -#define ONFI_TIMING_MODE_1 (1 << 1) -#define ONFI_TIMING_MODE_2 (1 << 2) -#define ONFI_TIMING_MODE_3 (1 << 3) -#define ONFI_TIMING_MODE_4 (1 << 4) -#define ONFI_TIMING_MODE_5 (1 << 5) -#define ONFI_TIMING_MODE_UNKNOWN (1 << 6) - -/* ONFI feature number/address */ -#define ONFI_FEATURE_NUMBER 256 -#define ONFI_FEATURE_ADDR_TIMING_MODE 0x1 - -/* Vendor-specific feature address (Micron) */ -#define ONFI_FEATURE_ADDR_READ_RETRY 0x89 -#define ONFI_FEATURE_ON_DIE_ECC 0x90 -#define ONFI_FEATURE_ON_DIE_ECC_EN BIT(3) - -/* ONFI subfeature parameters length */ -#define ONFI_SUBFEATURE_PARAM_LEN 4 - -/* ONFI optional commands SET/GET FEATURES supported? */ -#define ONFI_OPT_CMD_SET_GET_FEATURES (1 << 2) - -struct nand_onfi_params { - /* rev info and features block */ - /* 'O' 'N' 'F' 'I' */ - u8 sig[4]; - __le16 revision; - __le16 features; - __le16 opt_cmd; - u8 reserved0[2]; - __le16 ext_param_page_length; /* since ONFI 2.1 */ - u8 num_of_param_pages; /* since ONFI 2.1 */ - u8 reserved1[17]; - - /* manufacturer information block */ - char manufacturer[12]; - char model[20]; - u8 jedec_id; - __le16 date_code; - u8 reserved2[13]; - - /* memory organization block */ - __le32 byte_per_page; - __le16 spare_bytes_per_page; - __le32 data_bytes_per_ppage; - __le16 spare_bytes_per_ppage; - __le32 pages_per_block; - __le32 blocks_per_lun; - u8 lun_count; - u8 addr_cycles; - u8 bits_per_cell; - __le16 bb_per_lun; - __le16 block_endurance; - u8 guaranteed_good_blocks; - __le16 guaranteed_block_endurance; - u8 programs_per_page; - u8 ppage_attr; - u8 ecc_bits; - u8 interleaved_bits; - u8 interleaved_ops; - u8 reserved3[13]; - - /* electrical parameter block */ - u8 io_pin_capacitance_max; - __le16 async_timing_mode; - __le16 program_cache_timing_mode; - __le16 t_prog; - __le16 t_bers; - __le16 t_r; - __le16 t_ccs; - __le16 src_sync_timing_mode; - u8 src_ssync_features; - __le16 clk_pin_capacitance_typ; - __le16 io_pin_capacitance_typ; - __le16 input_pin_capacitance_typ; - u8 input_pin_capacitance_max; - u8 driver_strength_support; - __le16 t_int_r; - __le16 t_adl; - u8 reserved4[8]; - - /* vendor */ - __le16 vendor_revision; - u8 vendor[88]; - - __le16 crc; -} __packed; - -#define ONFI_CRC_BASE 0x4F4E - -/* Extended ECC information Block Definition (since ONFI 2.1) */ -struct onfi_ext_ecc_info { - u8 ecc_bits; - u8 codeword_size; - __le16 bb_per_lun; - __le16 block_endurance; - u8 reserved[2]; -} __packed; - -#define ONFI_SECTION_TYPE_0 0 /* Unused section. */ -#define ONFI_SECTION_TYPE_1 1 /* for additional sections. */ -#define ONFI_SECTION_TYPE_2 2 /* for ECC information. */ -struct onfi_ext_section { - u8 type; - u8 length; -} __packed; - -#define ONFI_EXT_SECTION_MAX 8 - -/* Extended Parameter Page Definition (since ONFI 2.1) */ -struct onfi_ext_param_page { - __le16 crc; - u8 sig[4]; /* 'E' 'P' 'P' 'S' */ - u8 reserved0[10]; - struct onfi_ext_section sections[ONFI_EXT_SECTION_MAX]; - - /* - * The actual size of the Extended Parameter Page is in - * @ext_param_page_length of nand_onfi_params{}. - * The following are the variable length sections. - * So we do not add any fields below. Please see the ONFI spec. - */ -} __packed; - -struct jedec_ecc_info { - u8 ecc_bits; - u8 codeword_size; - __le16 bb_per_lun; - __le16 block_endurance; - u8 reserved[2]; -} __packed; - -/* JEDEC features */ -#define JEDEC_FEATURE_16_BIT_BUS (1 << 0) - -struct nand_jedec_params { - /* rev info and features block */ - /* 'J' 'E' 'S' 'D' */ - u8 sig[4]; - __le16 revision; - __le16 features; - u8 opt_cmd[3]; - __le16 sec_cmd; - u8 num_of_param_pages; - u8 reserved0[18]; - - /* manufacturer information block */ - char manufacturer[12]; - char model[20]; - u8 jedec_id[6]; - u8 reserved1[10]; - - /* memory organization block */ - __le32 byte_per_page; - __le16 spare_bytes_per_page; - u8 reserved2[6]; - __le32 pages_per_block; - __le32 blocks_per_lun; - u8 lun_count; - u8 addr_cycles; - u8 bits_per_cell; - u8 programs_per_page; - u8 multi_plane_addr; - u8 multi_plane_op_attr; - u8 reserved3[38]; - - /* electrical parameter block */ - __le16 async_sdr_speed_grade; - __le16 toggle_ddr_speed_grade; - __le16 sync_ddr_speed_grade; - u8 async_sdr_features; - u8 toggle_ddr_features; - u8 sync_ddr_features; - __le16 t_prog; - __le16 t_bers; - __le16 t_r; - __le16 t_r_multi_plane; - __le16 t_ccs; - __le16 io_pin_capacitance_typ; - __le16 input_pin_capacitance_typ; - __le16 clk_pin_capacitance_typ; - u8 driver_strength_support; - __le16 t_adl; - u8 reserved4[36]; - - /* ECC and endurance block */ - u8 guaranteed_good_blocks; - __le16 guaranteed_block_endurance; - struct jedec_ecc_info ecc_info[4]; - u8 reserved5[29]; - - /* reserved */ - u8 reserved6[148]; - - /* vendor */ - __le16 vendor_rev_num; - u8 reserved7[88]; - - /* CRC for Parameter Page */ - __le16 crc; -} __packed; - -/** - * struct onfi_params - ONFI specific parameters that will be reused - * @version: ONFI version (BCD encoded), 0 if ONFI is not supported - * @tPROG: Page program time - * @tBERS: Block erase time - * @tR: Page read time - * @tCCS: Change column setup time - * @async_timing_mode: Supported asynchronous timing mode - * @vendor_revision: Vendor specific revision number - * @vendor: Vendor specific data - */ -struct onfi_params { - int version; - u16 tPROG; - u16 tBERS; - u16 tR; - u16 tCCS; - u16 async_timing_mode; - u16 vendor_revision; - u8 vendor[88]; -}; - /** * struct nand_parameters - NAND generic parameters from the parameter page * @model: Model name @@ -646,31 +392,28 @@ struct nand_ecc_ctrl { void *priv; u8 *calc_buf; u8 *code_buf; - void (*hwctl)(struct mtd_info *mtd, int mode); - int (*calculate)(struct mtd_info *mtd, const uint8_t *dat, - uint8_t *ecc_code); - int (*correct)(struct mtd_info *mtd, uint8_t *dat, uint8_t *read_ecc, - uint8_t *calc_ecc); - int (*read_page_raw)(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page); - int (*write_page_raw)(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, int page); - int (*read_page)(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page); - int (*read_subpage)(struct mtd_info *mtd, struct nand_chip *chip, - uint32_t offs, uint32_t len, uint8_t *buf, int page); - int (*write_subpage)(struct mtd_info *mtd, struct nand_chip *chip, - uint32_t offset, uint32_t data_len, - const uint8_t *data_buf, int oob_required, int page); - int (*write_page)(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, int page); - int (*write_oob_raw)(struct mtd_info *mtd, struct nand_chip *chip, - int page); - int (*read_oob_raw)(struct mtd_info *mtd, struct nand_chip *chip, - int page); - int (*read_oob)(struct mtd_info *mtd, struct nand_chip *chip, int page); - int (*write_oob)(struct mtd_info *mtd, struct nand_chip *chip, - int page); + void (*hwctl)(struct nand_chip *chip, int mode); + int (*calculate)(struct nand_chip *chip, const uint8_t *dat, + uint8_t *ecc_code); + int (*correct)(struct nand_chip *chip, uint8_t *dat, uint8_t *read_ecc, + uint8_t *calc_ecc); + int (*read_page_raw)(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page); + int (*write_page_raw)(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page); + int (*read_page)(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page); + int (*read_subpage)(struct nand_chip *chip, uint32_t offs, + uint32_t len, uint8_t *buf, int page); + int (*write_subpage)(struct nand_chip *chip, uint32_t offset, + uint32_t data_len, const uint8_t *data_buf, + int oob_required, int page); + int (*write_page)(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page); + int (*write_oob_raw)(struct nand_chip *chip, int page); + int (*read_oob_raw)(struct nand_chip *chip, int page); + int (*read_oob)(struct nand_chip *chip, int page); + int (*write_oob)(struct nand_chip *chip, int page); }; /** @@ -800,24 +543,6 @@ nand_get_sdr_timings(const struct nand_data_interface *conf) } /** - * struct nand_manufacturer_ops - NAND Manufacturer operations - * @detect: detect the NAND memory organization and capabilities - * @init: initialize all vendor specific fields (like the ->read_retry() - * implementation) if any. - * @cleanup: the ->init() function may have allocated resources, ->cleanup() - * is here to let vendor specific code release those resources. - * @fixup_onfi_param_page: apply vendor specific fixups to the ONFI parameter - * page. This is called after the checksum is verified. - */ -struct nand_manufacturer_ops { - void (*detect)(struct nand_chip *chip); - int (*init)(struct nand_chip *chip); - void (*cleanup)(struct nand_chip *chip); - void (*fixup_onfi_param_page)(struct nand_chip *chip, - struct nand_onfi_params *p); -}; - -/** * struct nand_op_cmd_instr - Definition of a command instruction * @opcode: the command to issue in one cycle */ @@ -1175,44 +900,72 @@ int nand_op_parser_exec_op(struct nand_chip *chip, const struct nand_operation *op, bool check_only); /** + * struct nand_legacy - NAND chip legacy fields/hooks + * @IO_ADDR_R: address to read the 8 I/O lines of the flash device + * @IO_ADDR_W: address to write the 8 I/O lines of the flash device + * @read_byte: read one byte from the chip + * @write_byte: write a single byte to the chip on the low 8 I/O lines + * @write_buf: write data from the buffer to the chip + * @read_buf: read data from the chip into the buffer + * @cmd_ctrl: hardware specific function for controlling ALE/CLE/nCE. Also used + * to write command and address + * @cmdfunc: hardware specific function for writing commands to the chip. + * @dev_ready: hardware specific function for accessing device ready/busy line. + * If set to NULL no access to ready/busy is available and the + * ready/busy information is read from the chip status register. + * @waitfunc: hardware specific function for wait on ready. + * @block_bad: check if a block is bad, using OOB markers + * @block_markbad: mark a block bad + * @erase: erase function + * @set_features: set the NAND chip features + * @get_features: get the NAND chip features + * @chip_delay: chip dependent delay for transferring data from array to read + * regs (tR). + * + * If you look at this structure you're already wrong. These fields/hooks are + * all deprecated. + */ +struct nand_legacy { + void __iomem *IO_ADDR_R; + void __iomem *IO_ADDR_W; + u8 (*read_byte)(struct nand_chip *chip); + void (*write_byte)(struct nand_chip *chip, u8 byte); + void (*write_buf)(struct nand_chip *chip, const u8 *buf, int len); + void (*read_buf)(struct nand_chip *chip, u8 *buf, int len); + void (*cmd_ctrl)(struct nand_chip *chip, int dat, unsigned int ctrl); + void (*cmdfunc)(struct nand_chip *chip, unsigned command, int column, + int page_addr); + int (*dev_ready)(struct nand_chip *chip); + int (*waitfunc)(struct nand_chip *chip); + int (*block_bad)(struct nand_chip *chip, loff_t ofs); + int (*block_markbad)(struct nand_chip *chip, loff_t ofs); + int (*erase)(struct nand_chip *chip, int page); + int (*set_features)(struct nand_chip *chip, int feature_addr, + u8 *subfeature_para); + int (*get_features)(struct nand_chip *chip, int feature_addr, + u8 *subfeature_para); + int chip_delay; +}; + +/** * struct nand_chip - NAND Private Flash Chip Data * @mtd: MTD device registered to the MTD framework - * @IO_ADDR_R: [BOARDSPECIFIC] address to read the 8 I/O lines of the - * flash device - * @IO_ADDR_W: [BOARDSPECIFIC] address to write the 8 I/O lines of the - * flash device. - * @read_byte: [REPLACEABLE] read one byte from the chip - * @read_word: [REPLACEABLE] read one word from the chip - * @write_byte: [REPLACEABLE] write a single byte to the chip on the - * low 8 I/O lines - * @write_buf: [REPLACEABLE] write data from the buffer to the chip - * @read_buf: [REPLACEABLE] read data from the chip into the buffer + * @legacy: All legacy fields/hooks. If you develop a new driver, + * don't even try to use any of these fields/hooks, and if + * you're modifying an existing driver that is using those + * fields/hooks, you should consider reworking the driver + * avoid using them. * @select_chip: [REPLACEABLE] select chip nr - * @block_bad: [REPLACEABLE] check if a block is bad, using OOB markers - * @block_markbad: [REPLACEABLE] mark a block bad - * @cmd_ctrl: [BOARDSPECIFIC] hardwarespecific function for controlling - * ALE/CLE/nCE. Also used to write command and address - * @dev_ready: [BOARDSPECIFIC] hardwarespecific function for accessing - * device ready/busy line. If set to NULL no access to - * ready/busy is available and the ready/busy information - * is read from the chip status register. - * @cmdfunc: [REPLACEABLE] hardwarespecific function for writing - * commands to the chip. - * @waitfunc: [REPLACEABLE] hardwarespecific function for wait on - * ready. * @exec_op: controller specific method to execute NAND operations. * This method replaces ->cmdfunc(), - * ->{read,write}_{buf,byte,word}(), ->dev_ready() and - * ->waifunc(). + * ->legacy.{read,write}_{buf,byte,word}(), + * ->legacy.dev_ready() and ->waifunc(). * @setup_read_retry: [FLASHSPECIFIC] flash (vendor) specific function for * setting the read-retry mode. Mostly needed for MLC NAND. * @ecc: [BOARDSPECIFIC] ECC control structure * @buf_align: minimum buffer alignment required by a platform * @dummy_controller: dummy controller implementation for drivers that can * only control a single chip - * @erase: [REPLACEABLE] erase function - * @chip_delay: [BOARDSPECIFIC] chip dependent delay for transferring - * data from array to read regs (tR). * @state: [INTERN] the current state of the NAND device * @oob_poi: "poison value buffer," used for laying out OOB data * before writing @@ -1260,8 +1013,6 @@ int nand_op_parser_exec_op(struct nand_chip *chip, * @blocks_per_die: [INTERN] The number of PEBs in a die * @data_interface: [INTERN] NAND interface timing information * @read_retries: [INTERN] the number of read retry modes supported - * @set_features: [REPLACEABLE] set the NAND chip features - * @get_features: [REPLACEABLE] get the NAND chip features * @setup_data_interface: [OPTIONAL] setup the data interface and timing. If * chipnr is set to %NAND_DATA_IFACE_CHECK_ONLY this * means the configuration should not be applied but @@ -1283,35 +1034,17 @@ int nand_op_parser_exec_op(struct nand_chip *chip, struct nand_chip { struct mtd_info mtd; - void __iomem *IO_ADDR_R; - void __iomem *IO_ADDR_W; - uint8_t (*read_byte)(struct mtd_info *mtd); - u16 (*read_word)(struct mtd_info *mtd); - void (*write_byte)(struct mtd_info *mtd, uint8_t byte); - void (*write_buf)(struct mtd_info *mtd, const uint8_t *buf, int len); - void (*read_buf)(struct mtd_info *mtd, uint8_t *buf, int len); - void (*select_chip)(struct mtd_info *mtd, int chip); - int (*block_bad)(struct mtd_info *mtd, loff_t ofs); - int (*block_markbad)(struct mtd_info *mtd, loff_t ofs); - void (*cmd_ctrl)(struct mtd_info *mtd, int dat, unsigned int ctrl); - int (*dev_ready)(struct mtd_info *mtd); - void (*cmdfunc)(struct mtd_info *mtd, unsigned command, int column, - int page_addr); - int(*waitfunc)(struct mtd_info *mtd, struct nand_chip *this); + struct nand_legacy legacy; + + void (*select_chip)(struct nand_chip *chip, int cs); int (*exec_op)(struct nand_chip *chip, const struct nand_operation *op, bool check_only); - int (*erase)(struct mtd_info *mtd, int page); - int (*set_features)(struct mtd_info *mtd, struct nand_chip *chip, - int feature_addr, uint8_t *subfeature_para); - int (*get_features)(struct mtd_info *mtd, struct nand_chip *chip, - int feature_addr, uint8_t *subfeature_para); - int (*setup_read_retry)(struct mtd_info *mtd, int retry_mode); - int (*setup_data_interface)(struct mtd_info *mtd, int chipnr, + int (*setup_read_retry)(struct nand_chip *chip, int retry_mode); + int (*setup_data_interface)(struct nand_chip *chip, int chipnr, const struct nand_data_interface *conf); - int chip_delay; unsigned int options; unsigned int bbt_options; @@ -1420,27 +1153,6 @@ static inline void *nand_get_manufacturer_data(struct nand_chip *chip) } /* - * NAND Flash Manufacturer ID Codes - */ -#define NAND_MFR_TOSHIBA 0x98 -#define NAND_MFR_ESMT 0xc8 -#define NAND_MFR_SAMSUNG 0xec -#define NAND_MFR_FUJITSU 0x04 -#define NAND_MFR_NATIONAL 0x8f -#define NAND_MFR_RENESAS 0x07 -#define NAND_MFR_STMICRO 0x20 -#define NAND_MFR_HYNIX 0xad -#define NAND_MFR_MICRON 0x2c -#define NAND_MFR_AMD 0x01 -#define NAND_MFR_MACRONIX 0xc2 -#define NAND_MFR_EON 0x92 -#define NAND_MFR_SANDISK 0x45 -#define NAND_MFR_INTEL 0x89 -#define NAND_MFR_ATO 0x9b -#define NAND_MFR_WINBOND 0xef - - -/* * A helper for defining older NAND chips where the second ID byte fully * defined the chip, including the geometry (chip size, eraseblock size, page * size). All these chips have 512 bytes NAND page size. @@ -1519,114 +1231,7 @@ struct nand_flash_dev { int onfi_timing_mode_default; }; -/** - * struct nand_manufacturer - NAND Flash Manufacturer structure - * @name: Manufacturer name - * @id: manufacturer ID code of device. - * @ops: manufacturer operations -*/ -struct nand_manufacturer { - int id; - char *name; - const struct nand_manufacturer_ops *ops; -}; - -const struct nand_manufacturer *nand_get_manufacturer(u8 id); - -static inline const char * -nand_manufacturer_name(const struct nand_manufacturer *manufacturer) -{ - return manufacturer ? manufacturer->name : "Unknown"; -} - -extern struct nand_flash_dev nand_flash_ids[]; - -extern const struct nand_manufacturer_ops toshiba_nand_manuf_ops; -extern const struct nand_manufacturer_ops samsung_nand_manuf_ops; -extern const struct nand_manufacturer_ops hynix_nand_manuf_ops; -extern const struct nand_manufacturer_ops micron_nand_manuf_ops; -extern const struct nand_manufacturer_ops amd_nand_manuf_ops; -extern const struct nand_manufacturer_ops macronix_nand_manuf_ops; - int nand_create_bbt(struct nand_chip *chip); -int nand_markbad_bbt(struct mtd_info *mtd, loff_t offs); -int nand_isreserved_bbt(struct mtd_info *mtd, loff_t offs); -int nand_isbad_bbt(struct mtd_info *mtd, loff_t offs, int allowbbt); -int nand_erase_nand(struct mtd_info *mtd, struct erase_info *instr, - int allowbbt); - -/** - * struct platform_nand_chip - chip level device structure - * @nr_chips: max. number of chips to scan for - * @chip_offset: chip number offset - * @nr_partitions: number of partitions pointed to by partitions (or zero) - * @partitions: mtd partition list - * @chip_delay: R/B delay value in us - * @options: Option flags, e.g. 16bit buswidth - * @bbt_options: BBT option flags, e.g. NAND_BBT_USE_FLASH - * @part_probe_types: NULL-terminated array of probe types - */ -struct platform_nand_chip { - int nr_chips; - int chip_offset; - int nr_partitions; - struct mtd_partition *partitions; - int chip_delay; - unsigned int options; - unsigned int bbt_options; - const char **part_probe_types; -}; - -/* Keep gcc happy */ -struct platform_device; - -/** - * struct platform_nand_ctrl - controller level device structure - * @probe: platform specific function to probe/setup hardware - * @remove: platform specific function to remove/teardown hardware - * @dev_ready: platform specific function to read ready/busy pin - * @select_chip: platform specific chip select function - * @cmd_ctrl: platform specific function for controlling - * ALE/CLE/nCE. Also used to write command and address - * @write_buf: platform specific function for write buffer - * @read_buf: platform specific function for read buffer - * @priv: private data to transport driver specific settings - * - * All fields are optional and depend on the hardware driver requirements - */ -struct platform_nand_ctrl { - int (*probe)(struct platform_device *pdev); - void (*remove)(struct platform_device *pdev); - int (*dev_ready)(struct mtd_info *mtd); - void (*select_chip)(struct mtd_info *mtd, int chip); - void (*cmd_ctrl)(struct mtd_info *mtd, int dat, unsigned int ctrl); - void (*write_buf)(struct mtd_info *mtd, const uint8_t *buf, int len); - void (*read_buf)(struct mtd_info *mtd, uint8_t *buf, int len); - void *priv; -}; - -/** - * struct platform_nand_data - container structure for platform-specific data - * @chip: chip level chip structure - * @ctrl: controller level device structure - */ -struct platform_nand_data { - struct platform_nand_chip chip; - struct platform_nand_ctrl ctrl; -}; - -/* return the supported asynchronous timing mode. */ -static inline int onfi_get_async_timing_mode(struct nand_chip *chip) -{ - if (!chip->parameters.onfi) - return ONFI_TIMING_MODE_UNKNOWN; - - return chip->parameters.onfi->async_timing_mode; -} - -int onfi_fill_data_interface(struct nand_chip *chip, - enum nand_data_interface_type type, - int timing_mode); /* * Check if it is a SLC nand. @@ -1658,9 +1263,6 @@ static inline int nand_opcode_8bits(unsigned int command) return 0; } -/* get timing characteristics from ONFI timing mode. */ -const struct nand_sdr_timings *onfi_async_timing_mode_to_sdr_timings(int mode); - int nand_check_erased_ecc_chunk(void *data, int datalen, void *ecc, int ecclen, void *extraoob, int extraooblen, @@ -1670,37 +1272,22 @@ int nand_ecc_choose_conf(struct nand_chip *chip, const struct nand_ecc_caps *caps, int oobavail); /* Default write_oob implementation */ -int nand_write_oob_std(struct mtd_info *mtd, struct nand_chip *chip, int page); - -/* Default write_oob syndrome implementation */ -int nand_write_oob_syndrome(struct mtd_info *mtd, struct nand_chip *chip, - int page); +int nand_write_oob_std(struct nand_chip *chip, int page); /* Default read_oob implementation */ -int nand_read_oob_std(struct mtd_info *mtd, struct nand_chip *chip, int page); +int nand_read_oob_std(struct nand_chip *chip, int page); -/* Default read_oob syndrome implementation */ -int nand_read_oob_syndrome(struct mtd_info *mtd, struct nand_chip *chip, - int page); - -/* Wrapper to use in order for controllers/vendors to GET/SET FEATURES */ -int nand_get_features(struct nand_chip *chip, int addr, u8 *subfeature_param); -int nand_set_features(struct nand_chip *chip, int addr, u8 *subfeature_param); /* Stub used by drivers that do not support GET/SET FEATURES operations */ -int nand_get_set_features_notsupp(struct mtd_info *mtd, struct nand_chip *chip, - int addr, u8 *subfeature_param); +int nand_get_set_features_notsupp(struct nand_chip *chip, int addr, + u8 *subfeature_param); /* Default read_page_raw implementation */ -int nand_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip, - uint8_t *buf, int oob_required, int page); -int nand_read_page_raw_notsupp(struct mtd_info *mtd, struct nand_chip *chip, - u8 *buf, int oob_required, int page); +int nand_read_page_raw(struct nand_chip *chip, uint8_t *buf, int oob_required, + int page); /* Default write_page_raw implementation */ -int nand_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip, - const uint8_t *buf, int oob_required, int page); -int nand_write_page_raw_notsupp(struct mtd_info *mtd, struct nand_chip *chip, - const u8 *buf, int oob_required, int page); +int nand_write_page_raw(struct nand_chip *chip, const uint8_t *buf, + int oob_required, int page); /* Reset and initialize a NAND device */ int nand_reset(struct nand_chip *chip, int chipnr); @@ -1710,7 +1297,6 @@ int nand_reset_op(struct nand_chip *chip); int nand_readid_op(struct nand_chip *chip, u8 addr, void *buf, unsigned int len); int nand_status_op(struct nand_chip *chip, u8 *status); -int nand_exit_status_op(struct nand_chip *chip); int nand_erase_op(struct nand_chip *chip, unsigned int eraseblock); int nand_read_page_op(struct nand_chip *chip, unsigned int page, unsigned int offset_in_page, void *buf, unsigned int len); @@ -1734,16 +1320,25 @@ int nand_read_data_op(struct nand_chip *chip, void *buf, unsigned int len, int nand_write_data_op(struct nand_chip *chip, const void *buf, unsigned int len, bool force_8bit); +/* Scan and identify a NAND device */ +int nand_scan_with_ids(struct nand_chip *chip, unsigned int max_chips, + struct nand_flash_dev *ids); + +static inline int nand_scan(struct nand_chip *chip, unsigned int max_chips) +{ + return nand_scan_with_ids(chip, max_chips, NULL); +} + +/* Internal helper for board drivers which need to override command function */ +void nand_wait_ready(struct nand_chip *chip); + /* * Free resources held by the NAND device, must be called on error after a * sucessful nand_scan(). */ void nand_cleanup(struct nand_chip *chip); /* Unregister the MTD device and calls nand_cleanup() */ -void nand_release(struct mtd_info *mtd); - -/* Default extended ID decoding function */ -void nand_decode_ext_id(struct nand_chip *chip); +void nand_release(struct nand_chip *chip); /* * External helper for controller drivers that have to implement the WAITRDY diff --git a/include/linux/mtd/spi-nor.h b/include/linux/mtd/spi-nor.h index c922e97f205a..7f0c7303575e 100644 --- a/include/linux/mtd/spi-nor.h +++ b/include/linux/mtd/spi-nor.h @@ -239,6 +239,94 @@ enum spi_nor_option_flags { }; /** + * struct spi_nor_erase_type - Structure to describe a SPI NOR erase type + * @size: the size of the sector/block erased by the erase type. + * JEDEC JESD216B imposes erase sizes to be a power of 2. + * @size_shift: @size is a power of 2, the shift is stored in + * @size_shift. + * @size_mask: the size mask based on @size_shift. + * @opcode: the SPI command op code to erase the sector/block. + * @idx: Erase Type index as sorted in the Basic Flash Parameter + * Table. It will be used to synchronize the supported + * Erase Types with the ones identified in the SFDP + * optional tables. + */ +struct spi_nor_erase_type { + u32 size; + u32 size_shift; + u32 size_mask; + u8 opcode; + u8 idx; +}; + +/** + * struct spi_nor_erase_command - Used for non-uniform erases + * The structure is used to describe a list of erase commands to be executed + * once we validate that the erase can be performed. The elements in the list + * are run-length encoded. + * @list: for inclusion into the list of erase commands. + * @count: how many times the same erase command should be + * consecutively used. + * @size: the size of the sector/block erased by the command. + * @opcode: the SPI command op code to erase the sector/block. + */ +struct spi_nor_erase_command { + struct list_head list; + u32 count; + u32 size; + u8 opcode; +}; + +/** + * struct spi_nor_erase_region - Structure to describe a SPI NOR erase region + * @offset: the offset in the data array of erase region start. + * LSB bits are used as a bitmask encoding flags to + * determine if this region is overlaid, if this region is + * the last in the SPI NOR flash memory and to indicate + * all the supported erase commands inside this region. + * The erase types are sorted in ascending order with the + * smallest Erase Type size being at BIT(0). + * @size: the size of the region in bytes. + */ +struct spi_nor_erase_region { + u64 offset; + u64 size; +}; + +#define SNOR_ERASE_TYPE_MAX 4 +#define SNOR_ERASE_TYPE_MASK GENMASK_ULL(SNOR_ERASE_TYPE_MAX - 1, 0) + +#define SNOR_LAST_REGION BIT(4) +#define SNOR_OVERLAID_REGION BIT(5) + +#define SNOR_ERASE_FLAGS_MAX 6 +#define SNOR_ERASE_FLAGS_MASK GENMASK_ULL(SNOR_ERASE_FLAGS_MAX - 1, 0) + +/** + * struct spi_nor_erase_map - Structure to describe the SPI NOR erase map + * @regions: array of erase regions. The regions are consecutive in + * address space. Walking through the regions is done + * incrementally. + * @uniform_region: a pre-allocated erase region for SPI NOR with a uniform + * sector size (legacy implementation). + * @erase_type: an array of erase types shared by all the regions. + * The erase types are sorted in ascending order, with the + * smallest Erase Type size being the first member in the + * erase_type array. + * @uniform_erase_type: bitmask encoding erase types that can erase the + * entire memory. This member is completed at init by + * uniform and non-uniform SPI NOR flash memories if they + * support at least one erase type that can erase the + * entire memory. + */ +struct spi_nor_erase_map { + struct spi_nor_erase_region *regions; + struct spi_nor_erase_region uniform_region; + struct spi_nor_erase_type erase_type[SNOR_ERASE_TYPE_MAX]; + u8 uniform_erase_type; +}; + +/** * struct flash_info - Forward declaration of a structure used internally by * spi_nor_scan() */ @@ -262,6 +350,7 @@ struct flash_info; * @write_proto: the SPI protocol for write operations * @reg_proto the SPI protocol for read_reg/write_reg/erase operations * @cmd_buf: used by the write_reg + * @erase_map: the erase map of the SPI NOR * @prepare: [OPTIONAL] do some preparations for the * read/write/erase/lock/unlock operations * @unprepare: [OPTIONAL] do some post work after the @@ -297,6 +386,7 @@ struct spi_nor { bool sst_write_second; u32 flags; u8 cmd_buf[SPI_NOR_MAX_CMD_SIZE]; + struct spi_nor_erase_map erase_map; int (*prepare)(struct spi_nor *nor, enum spi_nor_ops ops); void (*unprepare)(struct spi_nor *nor, enum spi_nor_ops ops); @@ -317,6 +407,35 @@ struct spi_nor { void *priv; }; +static u64 __maybe_unused +spi_nor_region_is_last(const struct spi_nor_erase_region *region) +{ + return region->offset & SNOR_LAST_REGION; +} + +static u64 __maybe_unused +spi_nor_region_end(const struct spi_nor_erase_region *region) +{ + return (region->offset & ~SNOR_ERASE_FLAGS_MASK) + region->size; +} + +static void __maybe_unused +spi_nor_region_mark_end(struct spi_nor_erase_region *region) +{ + region->offset |= SNOR_LAST_REGION; +} + +static void __maybe_unused +spi_nor_region_mark_overlay(struct spi_nor_erase_region *region) +{ + region->offset |= SNOR_OVERLAID_REGION; +} + +static bool __maybe_unused spi_nor_has_uniform_erase(const struct spi_nor *nor) +{ + return !!nor->erase_map.uniform_erase_type; +} + static inline void spi_nor_set_flash_node(struct spi_nor *nor, struct device_node *np) { diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 68e91ef5494c..818dbe9331be 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -1241,6 +1241,7 @@ enum { NVME_SC_ANA_PERSISTENT_LOSS = 0x301, NVME_SC_ANA_INACCESSIBLE = 0x302, NVME_SC_ANA_TRANSITION = 0x303, + NVME_SC_HOST_PATH_ERROR = 0x370, NVME_SC_DNR = 0x4000, }; diff --git a/include/linux/of_device.h b/include/linux/of_device.h index 165fd302b442..8d31e39dd564 100644 --- a/include/linux/of_device.h +++ b/include/linux/of_device.h @@ -58,7 +58,6 @@ static inline struct device_node *of_cpu_device_node_get(int cpu) int of_dma_configure(struct device *dev, struct device_node *np, bool force_dma); -void of_dma_deconfigure(struct device *dev); #else /* CONFIG_OF */ static inline int of_driver_match_device(struct device *dev, @@ -113,8 +112,6 @@ static inline int of_dma_configure(struct device *dev, { return 0; } -static inline void of_dma_deconfigure(struct device *dev) -{} #endif /* CONFIG_OF */ #endif /* _LINUX_OF_DEVICE_H */ diff --git a/include/linux/pci.h b/include/linux/pci.h index 6925828f9f25..2c4755032475 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1705,6 +1705,10 @@ static inline int pci_irqd_intx_xlate(struct irq_domain *d, unsigned long *out_hwirq, unsigned int *out_type) { return -EINVAL; } + +static inline const struct pci_device_id *pci_match_id(const struct pci_device_id *ids, + struct pci_dev *dev) +{ return NULL; } #endif /* CONFIG_PCI */ /* Include architecture-dependent settings and functions */ diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h index 009cdf3d65b6..b297cd1cd4f1 100644 --- a/include/linux/percpu-refcount.h +++ b/include/linux/percpu-refcount.h @@ -108,6 +108,7 @@ void percpu_ref_switch_to_atomic_sync(struct percpu_ref *ref); void percpu_ref_switch_to_percpu(struct percpu_ref *ref); void percpu_ref_kill_and_confirm(struct percpu_ref *ref, percpu_ref_func_t *confirm_kill); +void percpu_ref_resurrect(struct percpu_ref *ref); void percpu_ref_reinit(struct percpu_ref *ref); /** diff --git a/include/linux/platform_data/spi-davinci.h b/include/linux/platform_data/spi-davinci.h index f4edcb03c40c..0638fb6353bc 100644 --- a/include/linux/platform_data/spi-davinci.h +++ b/include/linux/platform_data/spi-davinci.h @@ -36,9 +36,6 @@ enum { * @num_chipselect: number of chipselects supported by this SPI master * @intr_line: interrupt line used to connect the SPI IP to the ARM interrupt * controller withn the SoC. Possible values are 0 and 1. - * @chip_sel: list of GPIOs which can act as chip-selects for the SPI. - * SPI_INTERN_CS denotes internal SPI chip-select. Not necessary - * to populate if all chip-selects are internal. * @cshold_bug: set this to true if the SPI controller on your chip requires * a write to CSHOLD bit in between transfers (like in DM355). * @dma_event_q: DMA event queue to use if SPI_IO_TYPE_DMA is used for any @@ -48,7 +45,6 @@ struct davinci_spi_platform_data { u8 version; u8 num_chipselect; u8 intr_line; - u8 *chip_sel; u8 prescaler_limit; bool cshold_bug; enum dma_event_q dma_event_q; diff --git a/include/linux/pxa2xx_ssp.h b/include/linux/pxa2xx_ssp.h index 13b4244d44c1..979087e021f3 100644 --- a/include/linux/pxa2xx_ssp.h +++ b/include/linux/pxa2xx_ssp.h @@ -196,6 +196,7 @@ enum pxa_ssp_type { PXA27x_SSP, PXA3xx_SSP, PXA168_SSP, + MMP2_SSP, PXA910_SSP, CE4100_SSP, QUARK_X1000_SSP, @@ -217,7 +218,7 @@ struct ssp_device { const char *label; int port_id; - int type; + enum pxa_ssp_type type; int use_count; int irq; diff --git a/include/linux/qcom-geni-se.h b/include/linux/qcom-geni-se.h index 5d6144977828..3bcd67fd5548 100644 --- a/include/linux/qcom-geni-se.h +++ b/include/linux/qcom-geni-se.h @@ -225,19 +225,14 @@ struct geni_se { #define HW_VER_MINOR_SHFT 16 #define HW_VER_STEP_MASK GENMASK(15, 0) +#define GENI_SE_VERSION_MAJOR(ver) ((ver & HW_VER_MAJOR_MASK) >> HW_VER_MAJOR_SHFT) +#define GENI_SE_VERSION_MINOR(ver) ((ver & HW_VER_MINOR_MASK) >> HW_VER_MINOR_SHFT) +#define GENI_SE_VERSION_STEP(ver) (ver & HW_VER_STEP_MASK) + #if IS_ENABLED(CONFIG_QCOM_GENI_SE) u32 geni_se_get_qup_hw_version(struct geni_se *se); -#define geni_se_get_wrapper_version(se, major, minor, step) do { \ - u32 ver; \ -\ - ver = geni_se_get_qup_hw_version(se); \ - major = (ver & HW_VER_MAJOR_MASK) >> HW_VER_MAJOR_SHFT; \ - minor = (ver & HW_VER_MINOR_MASK) >> HW_VER_MINOR_SHFT; \ - step = version & HW_VER_STEP_MASK; \ -} while (0) - /** * geni_se_read_proto() - Read the protocol configured for a serial engine * @se: Pointer to the concerned serial engine. diff --git a/include/linux/regmap.h b/include/linux/regmap.h index 379505a53722..a367d59c301d 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -268,6 +268,13 @@ typedef void (*regmap_unlock)(void *); * field is NULL but precious_table (see below) is not, the * check is performed on such table (a register is precious if * it belongs to one of the ranges specified by precious_table). + * @writeable_noinc_reg: Optional callback returning true if the register + * supports multiple write operations without incrementing + * the register number. If this field is NULL but + * wr_noinc_table (see below) is not, the check is + * performed on such table (a register is no increment + * writeable if it belongs to one of the ranges specified + * by wr_noinc_table). * @readable_noinc_reg: Optional callback returning true if the register * supports multiple read operations without incrementing * the register number. If this field is NULL but @@ -302,6 +309,7 @@ typedef void (*regmap_unlock)(void *); * @rd_table: As above, for read access. * @volatile_table: As above, for volatile registers. * @precious_table: As above, for precious registers. + * @wr_noinc_table: As above, for no increment writeable registers. * @rd_noinc_table: As above, for no increment readable registers. * @reg_defaults: Power on reset values for registers (for use with * register cache support). @@ -315,9 +323,12 @@ typedef void (*regmap_unlock)(void *); * masks are used. * @zero_flag_mask: If set, read_flag_mask and write_flag_mask are used even * if they are both empty. - * @use_single_rw: If set, converts the bulk read and write operations into - * a series of single read and write operations. This is useful - * for device that does not support bulk read and write. + * @use_single_read: If set, converts the bulk read operation into a series of + * single read operations. This is useful for a device that + * does not support bulk read. + * @use_single_write: If set, converts the bulk write operation into a series of + * single write operations. This is useful for a device that + * does not support bulk write. * @can_multi_write: If set, the device supports the multi write mode of bulk * write operations, if clear multi write requests will be * split into individual write operations @@ -352,6 +363,7 @@ struct regmap_config { bool (*readable_reg)(struct device *dev, unsigned int reg); bool (*volatile_reg)(struct device *dev, unsigned int reg); bool (*precious_reg)(struct device *dev, unsigned int reg); + bool (*writeable_noinc_reg)(struct device *dev, unsigned int reg); bool (*readable_noinc_reg)(struct device *dev, unsigned int reg); bool disable_locking; @@ -369,6 +381,7 @@ struct regmap_config { const struct regmap_access_table *rd_table; const struct regmap_access_table *volatile_table; const struct regmap_access_table *precious_table; + const struct regmap_access_table *wr_noinc_table; const struct regmap_access_table *rd_noinc_table; const struct reg_default *reg_defaults; unsigned int num_reg_defaults; @@ -380,7 +393,8 @@ struct regmap_config { unsigned long write_flag_mask; bool zero_flag_mask; - bool use_single_rw; + bool use_single_read; + bool use_single_write; bool can_multi_write; enum regmap_endian reg_format_endian; @@ -979,6 +993,8 @@ int regmap_write(struct regmap *map, unsigned int reg, unsigned int val); int regmap_write_async(struct regmap *map, unsigned int reg, unsigned int val); int regmap_raw_write(struct regmap *map, unsigned int reg, const void *val, size_t val_len); +int regmap_noinc_write(struct regmap *map, unsigned int reg, + const void *val, size_t val_len); int regmap_bulk_write(struct regmap *map, unsigned int reg, const void *val, size_t val_count); int regmap_multi_reg_write(struct regmap *map, const struct reg_sequence *regs, @@ -1222,6 +1238,13 @@ static inline int regmap_raw_write_async(struct regmap *map, unsigned int reg, return -EINVAL; } +static inline int regmap_noinc_write(struct regmap *map, unsigned int reg, + const void *val, size_t val_len) +{ + WARN_ONCE(1, "regmap API is disabled"); + return -EINVAL; +} + static inline int regmap_bulk_write(struct regmap *map, unsigned int reg, const void *val, size_t val_count) { diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index a64235e05321..6be77fa5ab90 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -1,15 +1,6 @@ -/* - * Copyright (C) 2005 David Brownell +/* SPDX-License-Identifier: GPL-2.0-or-later * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * Copyright (C) 2005 David Brownell */ #ifndef __LINUX_SPI_H @@ -163,10 +154,12 @@ struct spi_device { #define SPI_TX_QUAD 0x200 /* transmit with 4 wires */ #define SPI_RX_DUAL 0x400 /* receive with 2 wires */ #define SPI_RX_QUAD 0x800 /* receive with 4 wires */ +#define SPI_CS_WORD 0x1000 /* toggle cs after each word */ int irq; void *controller_state; void *controller_data; char modalias[SPI_NAME_SIZE]; + const char *driver_override; int cs_gpio; /* chip select gpio */ /* the statistics */ @@ -177,7 +170,6 @@ struct spi_device { * the controller talks to each chip, like: * - memory packing (12 bit samples into low bits, others zeroed) * - priority - * - drop chipselect after each word * - chipselect delays * - ... */ @@ -711,6 +703,8 @@ extern void spi_res_release(struct spi_controller *ctlr, * @delay_usecs: microseconds to delay after this transfer before * (optionally) changing the chipselect status, then starting * the next transfer or completing this @spi_message. + * @word_delay: clock cycles to inter word delay after each word size + * (set by bits_per_word) transmission. * @transfer_list: transfers are sequenced through @spi_message.transfers * @tx_sg: Scatterlist for transmit, currently not for client use * @rx_sg: Scatterlist for receive, currently not for client use @@ -793,6 +787,7 @@ struct spi_transfer { u8 bits_per_word; u16 delay_usecs; u32 speed_hz; + u16 word_delay; struct list_head transfer_list; }; @@ -1277,7 +1272,6 @@ spi_register_board_info(struct spi_board_info const *info, unsigned n) { return 0; } #endif - /* If you're hotplugging an adapter with devices (parport, usb, etc) * use spi_new_device() to describe each device. You can also call * spi_unregister_device() to start making that device vanish, but @@ -1309,6 +1303,22 @@ spi_transfer_is_last(struct spi_controller *ctlr, struct spi_transfer *xfer) return list_is_last(&xfer->transfer_list, &ctlr->cur_msg->transfers); } +/* OF support code */ +#if IS_ENABLED(CONFIG_OF) + +/* must call put_device() when done with returned spi_device device */ +extern struct spi_device * +of_find_spi_device_by_node(struct device_node *node); + +#else + +static inline struct spi_device * +of_find_spi_device_by_node(struct device_node *node) +{ + return NULL; +} + +#endif /* IS_ENABLED(CONFIG_OF) */ /* Compatibility layer */ #define spi_master spi_controller diff --git a/include/linux/tracepoint-defs.h b/include/linux/tracepoint-defs.h index 22c5a46e9693..49ba9cde7e4b 100644 --- a/include/linux/tracepoint-defs.h +++ b/include/linux/tracepoint-defs.h @@ -35,6 +35,12 @@ struct tracepoint { struct tracepoint_func __rcu *funcs; }; +#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS +typedef const int tracepoint_ptr_t; +#else +typedef struct tracepoint * const tracepoint_ptr_t; +#endif + struct bpf_raw_event_map { struct tracepoint *tp; void *bpf_func; diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h index 041f7e56a289..538ba1a58f5b 100644 --- a/include/linux/tracepoint.h +++ b/include/linux/tracepoint.h @@ -99,6 +99,29 @@ extern void syscall_unregfunc(void); #define TRACE_DEFINE_ENUM(x) #define TRACE_DEFINE_SIZEOF(x) +#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS +static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) +{ + return offset_to_ptr(p); +} + +#define __TRACEPOINT_ENTRY(name) \ + asm(" .section \"__tracepoints_ptrs\", \"a\" \n" \ + " .balign 4 \n" \ + " .long __tracepoint_" #name " - . \n" \ + " .previous \n") +#else +static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) +{ + return *p; +} + +#define __TRACEPOINT_ENTRY(name) \ + static tracepoint_ptr_t __tracepoint_ptr_##name __used \ + __attribute__((section("__tracepoints_ptrs"))) = \ + &__tracepoint_##name +#endif + #endif /* _LINUX_TRACEPOINT_H */ /* @@ -253,19 +276,6 @@ extern void syscall_unregfunc(void); return static_key_false(&__tracepoint_##name.key); \ } -#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS -#define __TRACEPOINT_ENTRY(name) \ - asm(" .section \"__tracepoints_ptrs\", \"a\" \n" \ - " .balign 4 \n" \ - " .long __tracepoint_" #name " - . \n" \ - " .previous \n") -#else -#define __TRACEPOINT_ENTRY(name) \ - static struct tracepoint * const __tracepoint_ptr_##name __used \ - __attribute__((section("__tracepoints_ptrs"))) = \ - &__tracepoint_##name -#endif - /* * We have no guarantee that gcc and the linker won't up-align the tracepoint * structures, so we create an array of pointers that will be used for iteration diff --git a/include/linux/writeback.h b/include/linux/writeback.h index fdfd04e348f6..738a0c24874f 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -246,7 +246,8 @@ static inline void wbc_attach_fdatawrite_inode(struct writeback_control *wbc, * * @bio is a part of the writeback in progress controlled by @wbc. Perform * writeback specific initialization. This is used to apply the cgroup - * writeback context. + * writeback context. Must be called after the bio has been associated with + * a device. */ static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio) { @@ -257,7 +258,7 @@ static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio) * regular writeback instead of writing things out itself. */ if (wbc->wb) - bio_associate_blkcg(bio, wbc->wb->blkcg_css); + bio_associate_blkg_from_css(bio, wbc->wb->blkcg_css); } #else /* CONFIG_CGROUP_WRITEBACK */ diff --git a/include/net/dst.h b/include/net/dst.h index 7f735e76ca73..6cf0870414c7 100644 --- a/include/net/dst.h +++ b/include/net/dst.h @@ -527,4 +527,14 @@ static inline void skb_dst_update_pmtu(struct sk_buff *skb, u32 mtu) dst->ops->update_pmtu(dst, NULL, skb, mtu); } +static inline void skb_tunnel_check_pmtu(struct sk_buff *skb, + struct dst_entry *encap_dst, + int headroom) +{ + u32 encap_mtu = dst_mtu(encap_dst); + + if (skb->len > encap_mtu - headroom) + skb_dst_update_pmtu(skb, encap_mtu - headroom); +} + #endif /* _NET_DST_H */ diff --git a/include/net/ip6_fib.h b/include/net/ip6_fib.h index 3d4930528db0..2d31e22babd8 100644 --- a/include/net/ip6_fib.h +++ b/include/net/ip6_fib.h @@ -159,6 +159,10 @@ struct fib6_info { struct rt6_info * __percpu *rt6i_pcpu; struct rt6_exception_bucket __rcu *rt6i_exception_bucket; +#ifdef CONFIG_IPV6_ROUTER_PREF + unsigned long last_probe; +#endif + u32 fib6_metric; u8 fib6_protocol; u8 fib6_type; diff --git a/include/net/sctp/sm.h b/include/net/sctp/sm.h index 5ef1bad81ef5..9e3d32746430 100644 --- a/include/net/sctp/sm.h +++ b/include/net/sctp/sm.h @@ -347,7 +347,7 @@ static inline __u16 sctp_data_size(struct sctp_chunk *chunk) __u16 size; size = ntohs(chunk->chunk_hdr->length); - size -= sctp_datahdr_len(&chunk->asoc->stream); + size -= sctp_datachk_len(&chunk->asoc->stream); return size; } diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h index 28a7c8e44636..a11f93790476 100644 --- a/include/net/sctp/structs.h +++ b/include/net/sctp/structs.h @@ -876,6 +876,8 @@ struct sctp_transport { unsigned long sackdelay; __u32 sackfreq; + atomic_t mtu_info; + /* When was the last time that we heard from this transport? We use * this to pick new active and retran paths. */ diff --git a/include/trace/events/hwmon.h b/include/trace/events/hwmon.h new file mode 100644 index 000000000000..d7a1d0ffb679 --- /dev/null +++ b/include/trace/events/hwmon.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM hwmon + +#if !defined(_TRACE_HWMON_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HWMON_H + +#include <linux/tracepoint.h> + +DECLARE_EVENT_CLASS(hwmon_attr_class, + + TP_PROTO(int index, const char *attr_name, long val), + + TP_ARGS(index, attr_name, val), + + TP_STRUCT__entry( + __field(int, index) + __string(attr_name, attr_name) + __field(long, val) + ), + + TP_fast_assign( + __entry->index = index; + __assign_str(attr_name, attr_name); + __entry->val = val; + ), + + TP_printk("index=%d, attr_name=%s, val=%ld", + __entry->index, __get_str(attr_name), __entry->val) +); + +DEFINE_EVENT(hwmon_attr_class, hwmon_attr_show, + + TP_PROTO(int index, const char *attr_name, long val), + + TP_ARGS(index, attr_name, val) +); + +DEFINE_EVENT(hwmon_attr_class, hwmon_attr_store, + + TP_PROTO(int index, const char *attr_name, long val), + + TP_ARGS(index, attr_name, val) +); + +TRACE_EVENT(hwmon_attr_show_string, + + TP_PROTO(int index, const char *attr_name, const char *s), + + TP_ARGS(index, attr_name, s), + + TP_STRUCT__entry( + __field(int, index) + __string(attr_name, attr_name) + __string(label, s) + ), + + TP_fast_assign( + __entry->index = index; + __assign_str(attr_name, attr_name); + __assign_str(label, s); + ), + + TP_printk("index=%d, attr_name=%s, val=%s", + __entry->index, __get_str(attr_name), __get_str(label)) +); + +#endif /* _TRACE_HWMON_H */ + +/* This part must be outside protection */ +#include <trace/define_trace.h> diff --git a/include/trace/events/kyber.h b/include/trace/events/kyber.h new file mode 100644 index 000000000000..a9834c37ac40 --- /dev/null +++ b/include/trace/events/kyber.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kyber + +#if !defined(_TRACE_KYBER_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_KYBER_H + +#include <linux/blkdev.h> +#include <linux/tracepoint.h> + +#define DOMAIN_LEN 16 +#define LATENCY_TYPE_LEN 8 + +TRACE_EVENT(kyber_latency, + + TP_PROTO(struct request_queue *q, const char *domain, const char *type, + unsigned int percentile, unsigned int numerator, + unsigned int denominator, unsigned int samples), + + TP_ARGS(q, domain, type, percentile, numerator, denominator, samples), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __array( char, domain, DOMAIN_LEN ) + __array( char, type, LATENCY_TYPE_LEN ) + __field( u8, percentile ) + __field( u8, numerator ) + __field( u8, denominator ) + __field( unsigned int, samples ) + ), + + TP_fast_assign( + __entry->dev = disk_devt(dev_to_disk(kobj_to_dev(q->kobj.parent))); + strlcpy(__entry->domain, domain, DOMAIN_LEN); + strlcpy(__entry->type, type, DOMAIN_LEN); + __entry->percentile = percentile; + __entry->numerator = numerator; + __entry->denominator = denominator; + __entry->samples = samples; + ), + + TP_printk("%d,%d %s %s p%u %u/%u samples=%u", + MAJOR(__entry->dev), MINOR(__entry->dev), __entry->domain, + __entry->type, __entry->percentile, __entry->numerator, + __entry->denominator, __entry->samples) +); + +TRACE_EVENT(kyber_adjust, + + TP_PROTO(struct request_queue *q, const char *domain, + unsigned int depth), + + TP_ARGS(q, domain, depth), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __array( char, domain, DOMAIN_LEN ) + __field( unsigned int, depth ) + ), + + TP_fast_assign( + __entry->dev = disk_devt(dev_to_disk(kobj_to_dev(q->kobj.parent))); + strlcpy(__entry->domain, domain, DOMAIN_LEN); + __entry->depth = depth; + ), + + TP_printk("%d,%d %s %u", + MAJOR(__entry->dev), MINOR(__entry->dev), __entry->domain, + __entry->depth) +); + +TRACE_EVENT(kyber_throttled, + + TP_PROTO(struct request_queue *q, const char *domain), + + TP_ARGS(q, domain), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __array( char, domain, DOMAIN_LEN ) + ), + + TP_fast_assign( + __entry->dev = disk_devt(dev_to_disk(kobj_to_dev(q->kobj.parent))); + strlcpy(__entry->domain, domain, DOMAIN_LEN); + ), + + TP_printk("%d,%d %s", MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->domain) +); + +#define _TRACE_KYBER_H +#endif /* _TRACE_KYBER_H */ + +/* This part must be outside protection */ +#include <trace/define_trace.h> diff --git a/include/uapi/linux/sctp.h b/include/uapi/linux/sctp.h index b479db5c71d9..34dd3d497f2c 100644 --- a/include/uapi/linux/sctp.h +++ b/include/uapi/linux/sctp.h @@ -301,6 +301,7 @@ enum sctp_sinfo_flags { SCTP_SACK_IMMEDIATELY = (1 << 3), /* SACK should be sent without delay. */ /* 2 bits here have been used by SCTP_PR_SCTP_MASK */ SCTP_SENDALL = (1 << 6), + SCTP_PR_SCTP_ALL = (1 << 7), SCTP_NOTIFICATION = MSG_NOTIFICATION, /* Next message is not user msg but notification. */ SCTP_EOF = MSG_FIN, /* Initiate graceful shutdown process. */ }; diff --git a/include/xen/xen.h b/include/xen/xen.h index 1e1d9bd0bd37..d7a2678da77f 100644 --- a/include/xen/xen.h +++ b/include/xen/xen.h @@ -39,4 +39,8 @@ extern uint32_t xen_start_flags; #define xen_initial_domain() (0) #endif /* CONFIG_XEN_DOM0 */ +struct bio_vec; +bool xen_biovec_phys_mergeable(const struct bio_vec *vec1, + const struct bio_vec *vec2); + #endif /* _XEN_XEN_H */ diff --git a/kernel/bpf/xskmap.c b/kernel/bpf/xskmap.c index 9f8463afda9c..47147c9e184d 100644 --- a/kernel/bpf/xskmap.c +++ b/kernel/bpf/xskmap.c @@ -192,11 +192,8 @@ static int xsk_map_update_elem(struct bpf_map *map, void *key, void *value, sock_hold(sock->sk); old_xs = xchg(&m->xsk_map[i], xs); - if (old_xs) { - /* Make sure we've flushed everything. */ - synchronize_net(); + if (old_xs) sock_put((struct sock *)old_xs); - } sockfd_put(sock); return 0; @@ -212,11 +209,8 @@ static int xsk_map_delete_elem(struct bpf_map *map, void *key) return -EINVAL; old_xs = xchg(&m->xsk_map[k], NULL); - if (old_xs) { - /* Make sure we've flushed everything. */ - synchronize_net(); + if (old_xs) sock_put((struct sock *)old_xs); - } return 0; } diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 4a3dae2a8283..4c1cf0969a80 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -492,7 +492,7 @@ static struct cgroup_subsys_state *cgroup_tryget_css(struct cgroup *cgrp, } /** - * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem + * cgroup_e_css_by_mask - obtain a cgroup's effective css for the specified ss * @cgrp: the cgroup of interest * @ss: the subsystem of interest (%NULL returns @cgrp->self) * @@ -501,8 +501,8 @@ static struct cgroup_subsys_state *cgroup_tryget_css(struct cgroup *cgrp, * enabled. If @ss is associated with the hierarchy @cgrp is on, this * function is guaranteed to return non-NULL css. */ -static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp, - struct cgroup_subsys *ss) +static struct cgroup_subsys_state *cgroup_e_css_by_mask(struct cgroup *cgrp, + struct cgroup_subsys *ss) { lockdep_assert_held(&cgroup_mutex); @@ -523,6 +523,35 @@ static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp, } /** + * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem + * @cgrp: the cgroup of interest + * @ss: the subsystem of interest + * + * Find and get the effective css of @cgrp for @ss. The effective css is + * defined as the matching css of the nearest ancestor including self which + * has @ss enabled. If @ss is not mounted on the hierarchy @cgrp is on, + * the root css is returned, so this function always returns a valid css. + * + * The returned css is not guaranteed to be online, and therefore it is the + * callers responsiblity to tryget a reference for it. + */ +struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp, + struct cgroup_subsys *ss) +{ + struct cgroup_subsys_state *css; + + do { + css = cgroup_css(cgrp, ss); + + if (css) + return css; + cgrp = cgroup_parent(cgrp); + } while (cgrp); + + return init_css_set.subsys[ss->id]; +} + +/** * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem * @cgrp: the cgroup of interest * @ss: the subsystem of interest @@ -604,10 +633,11 @@ EXPORT_SYMBOL_GPL(of_css); * * Should be called under cgroup_[tree_]mutex. */ -#define for_each_e_css(css, ssid, cgrp) \ - for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \ - if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \ - ; \ +#define for_each_e_css(css, ssid, cgrp) \ + for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \ + if (!((css) = cgroup_e_css_by_mask(cgrp, \ + cgroup_subsys[(ssid)]))) \ + ; \ else /** @@ -1006,7 +1036,7 @@ static struct css_set *find_existing_css_set(struct css_set *old_cset, * @ss is in this hierarchy, so we want the * effective css from @cgrp. */ - template[i] = cgroup_e_css(cgrp, ss); + template[i] = cgroup_e_css_by_mask(cgrp, ss); } else { /* * @ss is not in this hierarchy, so we don't want @@ -3023,7 +3053,7 @@ static int cgroup_apply_control(struct cgroup *cgrp) return ret; /* - * At this point, cgroup_e_css() results reflect the new csses + * At this point, cgroup_e_css_by_mask() results reflect the new csses * making the following cgroup_update_dfl_csses() properly update * css associations of all tasks in the subtree. */ diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig index 1b1d63b3634b..645c7a2ecde8 100644 --- a/kernel/dma/Kconfig +++ b/kernel/dma/Kconfig @@ -13,6 +13,9 @@ config NEED_DMA_MAP_STATE config ARCH_DMA_ADDR_T_64BIT def_bool 64BIT || PHYS_ADDR_T_64BIT +config ARCH_HAS_DMA_COHERENCE_H + bool + config HAVE_GENERIC_DMA_COHERENT bool @@ -26,22 +29,19 @@ config ARCH_HAS_SYNC_DMA_FOR_CPU config ARCH_HAS_SYNC_DMA_FOR_CPU_ALL bool -config DMA_DIRECT_OPS +config ARCH_HAS_DMA_COHERENT_TO_PFN bool - depends on HAS_DMA -config DMA_NONCOHERENT_OPS +config ARCH_HAS_DMA_MMAP_PGPROT bool - depends on HAS_DMA - select DMA_DIRECT_OPS -config DMA_NONCOHERENT_MMAP +config DMA_DIRECT_OPS bool - depends on DMA_NONCOHERENT_OPS + depends on HAS_DMA config DMA_NONCOHERENT_CACHE_SYNC bool - depends on DMA_NONCOHERENT_OPS + depends on DMA_DIRECT_OPS config DMA_VIRT_OPS bool diff --git a/kernel/dma/Makefile b/kernel/dma/Makefile index 6de44e4eb454..7d581e4eea4a 100644 --- a/kernel/dma/Makefile +++ b/kernel/dma/Makefile @@ -4,7 +4,6 @@ obj-$(CONFIG_HAS_DMA) += mapping.o obj-$(CONFIG_DMA_CMA) += contiguous.o obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += coherent.o obj-$(CONFIG_DMA_DIRECT_OPS) += direct.o -obj-$(CONFIG_DMA_NONCOHERENT_OPS) += noncoherent.o obj-$(CONFIG_DMA_VIRT_OPS) += virt.o obj-$(CONFIG_DMA_API_DEBUG) += debug.o obj-$(CONFIG_SWIOTLB) += swiotlb.o diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index 286d82329eb0..b2a87905846d 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -49,7 +49,11 @@ static phys_addr_t limit_cmdline; static int __init early_cma(char *p) { - pr_debug("%s(%s)\n", __func__, p); + if (!p) { + pr_err("Config string not provided\n"); + return -EINVAL; + } + size_cmdline = memparse(p, &p); if (*p != '@') return 0; diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index c007d25bee09..231ca4628062 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -1312,6 +1312,22 @@ static void check_sg_segment(struct device *dev, struct scatterlist *sg) #endif } +void debug_dma_map_single(struct device *dev, const void *addr, + unsigned long len) +{ + if (unlikely(dma_debug_disabled())) + return; + + if (!virt_addr_valid(addr)) + err_printk(dev, NULL, "DMA-API: device driver maps memory from invalid area [addr=%p] [len=%lu]\n", + addr, len); + + if (is_vmalloc_addr(addr)) + err_printk(dev, NULL, "DMA-API: device driver maps memory from vmalloc area [addr=%p] [len=%lu]\n", + addr, len); +} +EXPORT_SYMBOL(debug_dma_map_single); + void debug_dma_map_page(struct device *dev, struct page *page, size_t offset, size_t size, int direction, dma_addr_t dma_addr, bool map_single) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index de87b0282e74..87a6bc2a96c0 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -1,13 +1,16 @@ // SPDX-License-Identifier: GPL-2.0 /* - * DMA operations that map physical memory directly without using an IOMMU or - * flushing caches. + * Copyright (C) 2018 Christoph Hellwig. + * + * DMA operations that map physical memory directly without using an IOMMU. */ +#include <linux/bootmem.h> /* for max_pfn */ #include <linux/export.h> #include <linux/mm.h> #include <linux/dma-direct.h> #include <linux/scatterlist.h> #include <linux/dma-contiguous.h> +#include <linux/dma-noncoherent.h> #include <linux/pfn.h> #include <linux/set_memory.h> @@ -41,40 +44,83 @@ check_addr(struct device *dev, dma_addr_t dma_addr, size_t size, return false; } - if (*dev->dma_mask >= DMA_BIT_MASK(32)) { + if (*dev->dma_mask >= DMA_BIT_MASK(32) || dev->bus_dma_mask) { dev_err(dev, - "%s: overflow %pad+%zu of device mask %llx\n", - caller, &dma_addr, size, *dev->dma_mask); + "%s: overflow %pad+%zu of device mask %llx bus mask %llx\n", + caller, &dma_addr, size, + *dev->dma_mask, dev->bus_dma_mask); } return false; } return true; } +static inline dma_addr_t phys_to_dma_direct(struct device *dev, + phys_addr_t phys) +{ + if (force_dma_unencrypted()) + return __phys_to_dma(dev, phys); + return phys_to_dma(dev, phys); +} + +u64 dma_direct_get_required_mask(struct device *dev) +{ + u64 max_dma = phys_to_dma_direct(dev, (max_pfn - 1) << PAGE_SHIFT); + + if (dev->bus_dma_mask && dev->bus_dma_mask < max_dma) + max_dma = dev->bus_dma_mask; + + return (1ULL << (fls64(max_dma) - 1)) * 2 - 1; +} + +static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask, + u64 *phys_mask) +{ + if (dev->bus_dma_mask && dev->bus_dma_mask < dma_mask) + dma_mask = dev->bus_dma_mask; + + if (force_dma_unencrypted()) + *phys_mask = __dma_to_phys(dev, dma_mask); + else + *phys_mask = dma_to_phys(dev, dma_mask); + + /* + * Optimistically try the zone that the physical address mask falls + * into first. If that returns memory that isn't actually addressable + * we will fallback to the next lower zone and try again. + * + * Note that GFP_DMA32 and GFP_DMA are no ops without the corresponding + * zones. + */ + if (*phys_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS)) + return GFP_DMA; + if (*phys_mask <= DMA_BIT_MASK(32)) + return GFP_DMA32; + return 0; +} + static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) { - dma_addr_t addr = force_dma_unencrypted() ? - __phys_to_dma(dev, phys) : phys_to_dma(dev, phys); - return addr + size - 1 <= dev->coherent_dma_mask; + return phys_to_dma_direct(dev, phys) + size - 1 <= + min_not_zero(dev->coherent_dma_mask, dev->bus_dma_mask); } -void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, - gfp_t gfp, unsigned long attrs) +void *dma_direct_alloc_pages(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; int page_order = get_order(size); struct page *page = NULL; + u64 phys_mask; void *ret; + if (attrs & DMA_ATTR_NO_WARN) + gfp |= __GFP_NOWARN; + /* we always manually zero the memory once we are done: */ gfp &= ~__GFP_ZERO; - - /* GFP_DMA32 and GFP_DMA are no ops without the corresponding zones: */ - if (dev->coherent_dma_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS)) - gfp |= GFP_DMA; - if (dev->coherent_dma_mask <= DMA_BIT_MASK(32) && !(gfp & GFP_DMA)) - gfp |= GFP_DMA32; - + gfp |= __dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask, + &phys_mask); again: /* CMA can be used only in the context which permits sleeping */ if (gfpflags_allow_blocking(gfp)) { @@ -93,15 +139,14 @@ again: page = NULL; if (IS_ENABLED(CONFIG_ZONE_DMA32) && - dev->coherent_dma_mask < DMA_BIT_MASK(64) && + phys_mask < DMA_BIT_MASK(64) && !(gfp & (GFP_DMA32 | GFP_DMA))) { gfp |= GFP_DMA32; goto again; } if (IS_ENABLED(CONFIG_ZONE_DMA) && - dev->coherent_dma_mask < DMA_BIT_MASK(32) && - !(gfp & GFP_DMA)) { + phys_mask < DMA_BIT_MASK(32) && !(gfp & GFP_DMA)) { gfp = (gfp & ~GFP_DMA32) | GFP_DMA; goto again; } @@ -124,7 +169,7 @@ again: * NOTE: this function must never look at the dma_addr argument, because we want * to be able to use it as a helper for iommu implementations as well. */ -void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, +void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; @@ -136,14 +181,96 @@ void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, free_pages((unsigned long)cpu_addr, page_order); } +void *dma_direct_alloc(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) +{ + if (!dev_is_dma_coherent(dev)) + return arch_dma_alloc(dev, size, dma_handle, gfp, attrs); + return dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs); +} + +void dma_direct_free(struct device *dev, size_t size, + void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) +{ + if (!dev_is_dma_coherent(dev)) + arch_dma_free(dev, size, cpu_addr, dma_addr, attrs); + else + dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs); +} + +static void dma_direct_sync_single_for_device(struct device *dev, + dma_addr_t addr, size_t size, enum dma_data_direction dir) +{ + if (dev_is_dma_coherent(dev)) + return; + arch_sync_dma_for_device(dev, dma_to_phys(dev, addr), size, dir); +} + +static void dma_direct_sync_sg_for_device(struct device *dev, + struct scatterlist *sgl, int nents, enum dma_data_direction dir) +{ + struct scatterlist *sg; + int i; + + if (dev_is_dma_coherent(dev)) + return; + + for_each_sg(sgl, sg, nents, i) + arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir); +} + +#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ + defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) +static void dma_direct_sync_single_for_cpu(struct device *dev, + dma_addr_t addr, size_t size, enum dma_data_direction dir) +{ + if (dev_is_dma_coherent(dev)) + return; + arch_sync_dma_for_cpu(dev, dma_to_phys(dev, addr), size, dir); + arch_sync_dma_for_cpu_all(dev); +} + +static void dma_direct_sync_sg_for_cpu(struct device *dev, + struct scatterlist *sgl, int nents, enum dma_data_direction dir) +{ + struct scatterlist *sg; + int i; + + if (dev_is_dma_coherent(dev)) + return; + + for_each_sg(sgl, sg, nents, i) + arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir); + arch_sync_dma_for_cpu_all(dev); +} + +static void dma_direct_unmap_page(struct device *dev, dma_addr_t addr, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + dma_direct_sync_single_for_cpu(dev, addr, size, dir); +} + +static void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) +{ + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + dma_direct_sync_sg_for_cpu(dev, sgl, nents, dir); +} +#endif + dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long attrs) { - dma_addr_t dma_addr = phys_to_dma(dev, page_to_phys(page)) + offset; + phys_addr_t phys = page_to_phys(page) + offset; + dma_addr_t dma_addr = phys_to_dma(dev, phys); if (!check_addr(dev, dma_addr, size, __func__)) return DIRECT_MAPPING_ERROR; + + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + dma_direct_sync_single_for_device(dev, dma_addr, size, dir); return dma_addr; } @@ -162,31 +289,29 @@ int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents, sg_dma_len(sg) = sg->length; } + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + dma_direct_sync_sg_for_device(dev, sgl, nents, dir); return nents; } +/* + * Because 32-bit DMA masks are so common we expect every architecture to be + * able to satisfy them - either by not supporting more physical memory, or by + * providing a ZONE_DMA32. If neither is the case, the architecture needs to + * use an IOMMU instead of the direct mapping. + */ int dma_direct_supported(struct device *dev, u64 mask) { -#ifdef CONFIG_ZONE_DMA - if (mask < phys_to_dma(dev, DMA_BIT_MASK(ARCH_ZONE_DMA_BITS))) - return 0; -#else - /* - * Because 32-bit DMA masks are so common we expect every architecture - * to be able to satisfy them - either by not supporting more physical - * memory, or by providing a ZONE_DMA32. If neither is the case, the - * architecture needs to use an IOMMU instead of the direct mapping. - */ - if (mask < phys_to_dma(dev, DMA_BIT_MASK(32))) - return 0; -#endif - /* - * Upstream PCI/PCIe bridges or SoC interconnects may not carry - * as many DMA address bits as the device itself supports. - */ - if (dev->bus_dma_mask && mask > dev->bus_dma_mask) - return 0; - return 1; + u64 min_mask; + + if (IS_ENABLED(CONFIG_ZONE_DMA)) + min_mask = DMA_BIT_MASK(ARCH_ZONE_DMA_BITS); + else + min_mask = DMA_BIT_MASK(32); + + min_mask = min_t(u64, min_mask, (max_pfn - 1) << PAGE_SHIFT); + + return mask >= phys_to_dma(dev, min_mask); } int dma_direct_mapping_error(struct device *dev, dma_addr_t dma_addr) @@ -199,7 +324,20 @@ const struct dma_map_ops dma_direct_ops = { .free = dma_direct_free, .map_page = dma_direct_map_page, .map_sg = dma_direct_map_sg, +#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) + .sync_single_for_device = dma_direct_sync_single_for_device, + .sync_sg_for_device = dma_direct_sync_sg_for_device, +#endif +#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ + defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) + .sync_single_for_cpu = dma_direct_sync_single_for_cpu, + .sync_sg_for_cpu = dma_direct_sync_sg_for_cpu, + .unmap_page = dma_direct_unmap_page, + .unmap_sg = dma_direct_unmap_sg, +#endif + .get_required_mask = dma_direct_get_required_mask, .dma_supported = dma_direct_supported, .mapping_error = dma_direct_mapping_error, + .cache_sync = arch_dma_cache_sync, }; EXPORT_SYMBOL(dma_direct_ops); diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index d2a92ddaac4d..58dec7a92b7b 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -7,7 +7,7 @@ */ #include <linux/acpi.h> -#include <linux/dma-mapping.h> +#include <linux/dma-noncoherent.h> #include <linux/export.h> #include <linux/gfp.h> #include <linux/of_device.h> @@ -202,17 +202,26 @@ EXPORT_SYMBOL(dmam_release_declared_memory); * Create scatter-list for the already allocated DMA buffer. */ int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, - void *cpu_addr, dma_addr_t handle, size_t size) + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs) { - struct page *page = virt_to_page(cpu_addr); + struct page *page; int ret; - ret = sg_alloc_table(sgt, 1, GFP_KERNEL); - if (unlikely(ret)) - return ret; + if (!dev_is_dma_coherent(dev)) { + if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN)) + return -ENXIO; - sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); - return 0; + page = pfn_to_page(arch_dma_coherent_to_pfn(dev, cpu_addr, + dma_addr)); + } else { + page = virt_to_page(cpu_addr); + } + + ret = sg_alloc_table(sgt, 1, GFP_KERNEL); + if (!ret) + sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); + return ret; } EXPORT_SYMBOL(dma_common_get_sgtable); @@ -220,27 +229,37 @@ EXPORT_SYMBOL(dma_common_get_sgtable); * Create userspace mapping for the DMA-coherent memory. */ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size) + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs) { - int ret = -ENXIO; #ifndef CONFIG_ARCH_NO_COHERENT_DMA_MMAP unsigned long user_count = vma_pages(vma); unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; unsigned long off = vma->vm_pgoff; + unsigned long pfn; + int ret = -ENXIO; - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + vma->vm_page_prot = arch_dma_mmap_pgprot(dev, vma->vm_page_prot, attrs); if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) return ret; - if (off < count && user_count <= (count - off)) - ret = remap_pfn_range(vma, vma->vm_start, - page_to_pfn(virt_to_page(cpu_addr)) + off, - user_count << PAGE_SHIFT, - vma->vm_page_prot); -#endif /* !CONFIG_ARCH_NO_COHERENT_DMA_MMAP */ + if (off >= count || user_count > count - off) + return -ENXIO; - return ret; + if (!dev_is_dma_coherent(dev)) { + if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN)) + return -ENXIO; + pfn = arch_dma_coherent_to_pfn(dev, cpu_addr, dma_addr); + } else { + pfn = page_to_pfn(virt_to_page(cpu_addr)); + } + + return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, + user_count << PAGE_SHIFT, vma->vm_page_prot); +#else + return -ENXIO; +#endif /* !CONFIG_ARCH_NO_COHERENT_DMA_MMAP */ } EXPORT_SYMBOL(dma_common_mmap); @@ -327,19 +346,3 @@ void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags) vunmap(cpu_addr); } #endif - -/* - * enables DMA API use for a device - */ -int dma_configure(struct device *dev) -{ - if (dev->bus->dma_configure) - return dev->bus->dma_configure(dev); - return 0; -} - -void dma_deconfigure(struct device *dev) -{ - of_dma_deconfigure(dev); - acpi_dma_deconfigure(dev); -} diff --git a/kernel/dma/noncoherent.c b/kernel/dma/noncoherent.c deleted file mode 100644 index 031fe235d958..000000000000 --- a/kernel/dma/noncoherent.c +++ /dev/null @@ -1,106 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2018 Christoph Hellwig. - * - * DMA operations that map physical memory directly without providing cache - * coherence. - */ -#include <linux/export.h> -#include <linux/mm.h> -#include <linux/dma-direct.h> -#include <linux/dma-noncoherent.h> -#include <linux/scatterlist.h> - -static void dma_noncoherent_sync_single_for_device(struct device *dev, - dma_addr_t addr, size_t size, enum dma_data_direction dir) -{ - arch_sync_dma_for_device(dev, dma_to_phys(dev, addr), size, dir); -} - -static void dma_noncoherent_sync_sg_for_device(struct device *dev, - struct scatterlist *sgl, int nents, enum dma_data_direction dir) -{ - struct scatterlist *sg; - int i; - - for_each_sg(sgl, sg, nents, i) - arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir); -} - -static dma_addr_t dma_noncoherent_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction dir, - unsigned long attrs) -{ - dma_addr_t addr; - - addr = dma_direct_map_page(dev, page, offset, size, dir, attrs); - if (!dma_mapping_error(dev, addr) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - arch_sync_dma_for_device(dev, page_to_phys(page) + offset, - size, dir); - return addr; -} - -static int dma_noncoherent_map_sg(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, unsigned long attrs) -{ - nents = dma_direct_map_sg(dev, sgl, nents, dir, attrs); - if (nents > 0 && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - dma_noncoherent_sync_sg_for_device(dev, sgl, nents, dir); - return nents; -} - -#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ - defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) -static void dma_noncoherent_sync_single_for_cpu(struct device *dev, - dma_addr_t addr, size_t size, enum dma_data_direction dir) -{ - arch_sync_dma_for_cpu(dev, dma_to_phys(dev, addr), size, dir); - arch_sync_dma_for_cpu_all(dev); -} - -static void dma_noncoherent_sync_sg_for_cpu(struct device *dev, - struct scatterlist *sgl, int nents, enum dma_data_direction dir) -{ - struct scatterlist *sg; - int i; - - for_each_sg(sgl, sg, nents, i) - arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir); - arch_sync_dma_for_cpu_all(dev); -} - -static void dma_noncoherent_unmap_page(struct device *dev, dma_addr_t addr, - size_t size, enum dma_data_direction dir, unsigned long attrs) -{ - if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - dma_noncoherent_sync_single_for_cpu(dev, addr, size, dir); -} - -static void dma_noncoherent_unmap_sg(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, unsigned long attrs) -{ - if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - dma_noncoherent_sync_sg_for_cpu(dev, sgl, nents, dir); -} -#endif - -const struct dma_map_ops dma_noncoherent_ops = { - .alloc = arch_dma_alloc, - .free = arch_dma_free, - .mmap = arch_dma_mmap, - .sync_single_for_device = dma_noncoherent_sync_single_for_device, - .sync_sg_for_device = dma_noncoherent_sync_sg_for_device, - .map_page = dma_noncoherent_map_page, - .map_sg = dma_noncoherent_map_sg, -#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ - defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) - .sync_single_for_cpu = dma_noncoherent_sync_single_for_cpu, - .sync_sg_for_cpu = dma_noncoherent_sync_sg_for_cpu, - .unmap_page = dma_noncoherent_unmap_page, - .unmap_sg = dma_noncoherent_unmap_sg, -#endif - .dma_supported = dma_direct_supported, - .mapping_error = dma_direct_mapping_error, - .cache_sync = arch_dma_cache_sync, -}; -EXPORT_SYMBOL(dma_noncoherent_ops); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 7fc4a371bdd2..908c9cdae2f0 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4001,7 +4001,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) * put back on, and if we advance min_vruntime, we'll be placed back * further than we started -- ie. we'll be penalized. */ - if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE) + if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE) update_min_vruntime(cfs_rq); } @@ -4476,9 +4476,13 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) /* * Add to the _head_ of the list, so that an already-started - * distribute_cfs_runtime will not see us + * distribute_cfs_runtime will not see us. If disribute_cfs_runtime is + * not running add to the tail so that later runqueues don't get starved. */ - list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); + if (cfs_b->distribute_running) + list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); + else + list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); /* * If we're the first throttled task, make sure the bandwidth @@ -4622,14 +4626,16 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) * in us over-using our runtime if it is all used during this loop, but * only by limited amounts in that extreme case. */ - while (throttled && cfs_b->runtime > 0) { + while (throttled && cfs_b->runtime > 0 && !cfs_b->distribute_running) { runtime = cfs_b->runtime; + cfs_b->distribute_running = 1; raw_spin_unlock(&cfs_b->lock); /* we can't nest cfs_b->lock while distributing bandwidth */ runtime = distribute_cfs_runtime(cfs_b, runtime, runtime_expires); raw_spin_lock(&cfs_b->lock); + cfs_b->distribute_running = 0; throttled = !list_empty(&cfs_b->throttled_cfs_rq); cfs_b->runtime -= min(runtime, cfs_b->runtime); @@ -4740,6 +4746,11 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) /* confirm we're still not at a refresh boundary */ raw_spin_lock(&cfs_b->lock); + if (cfs_b->distribute_running) { + raw_spin_unlock(&cfs_b->lock); + return; + } + if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { raw_spin_unlock(&cfs_b->lock); return; @@ -4749,6 +4760,9 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) runtime = cfs_b->runtime; expires = cfs_b->runtime_expires; + if (runtime) + cfs_b->distribute_running = 1; + raw_spin_unlock(&cfs_b->lock); if (!runtime) @@ -4759,6 +4773,7 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) raw_spin_lock(&cfs_b->lock); if (expires == cfs_b->runtime_expires) cfs_b->runtime -= min(runtime, cfs_b->runtime); + cfs_b->distribute_running = 0; raw_spin_unlock(&cfs_b->lock); } @@ -4867,6 +4882,7 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) cfs_b->period_timer.function = sched_cfs_period_timer; hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); cfs_b->slack_timer.function = sched_cfs_slack_timer; + cfs_b->distribute_running = 0; } static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 455fa330de04..9683f458aec7 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -346,6 +346,8 @@ struct cfs_bandwidth { int nr_periods; int nr_throttled; u64 throttled_time; + + bool distribute_running; #endif }; diff --git a/kernel/signal.c b/kernel/signal.c index 5843c541fda9..e4aad0e90882 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -3460,7 +3460,8 @@ int do_sigaction(int sig, struct k_sigaction *act, struct k_sigaction *oact) } static int -do_sigaltstack (const stack_t *ss, stack_t *oss, unsigned long sp) +do_sigaltstack (const stack_t *ss, stack_t *oss, unsigned long sp, + size_t min_ss_size) { struct task_struct *t = current; @@ -3490,7 +3491,7 @@ do_sigaltstack (const stack_t *ss, stack_t *oss, unsigned long sp) ss_size = 0; ss_sp = NULL; } else { - if (unlikely(ss_size < MINSIGSTKSZ)) + if (unlikely(ss_size < min_ss_size)) return -ENOMEM; } @@ -3508,7 +3509,8 @@ SYSCALL_DEFINE2(sigaltstack,const stack_t __user *,uss, stack_t __user *,uoss) if (uss && copy_from_user(&new, uss, sizeof(stack_t))) return -EFAULT; err = do_sigaltstack(uss ? &new : NULL, uoss ? &old : NULL, - current_user_stack_pointer()); + current_user_stack_pointer(), + MINSIGSTKSZ); if (!err && uoss && copy_to_user(uoss, &old, sizeof(stack_t))) err = -EFAULT; return err; @@ -3519,7 +3521,8 @@ int restore_altstack(const stack_t __user *uss) stack_t new; if (copy_from_user(&new, uss, sizeof(stack_t))) return -EFAULT; - (void)do_sigaltstack(&new, NULL, current_user_stack_pointer()); + (void)do_sigaltstack(&new, NULL, current_user_stack_pointer(), + MINSIGSTKSZ); /* squash all but EFAULT for now */ return 0; } @@ -3553,7 +3556,8 @@ static int do_compat_sigaltstack(const compat_stack_t __user *uss_ptr, uss.ss_size = uss32.ss_size; } ret = do_sigaltstack(uss_ptr ? &uss : NULL, &uoss, - compat_user_stack_pointer()); + compat_user_stack_pointer(), + COMPAT_MINSIGSTKSZ); if (ret >= 0 && uoss_ptr) { compat_stack_t old; memset(&old, 0, sizeof(old)); diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index 2868d85f1fb1..fac0ddf8a8e2 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c @@ -764,9 +764,9 @@ blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) if (!bt || !(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP)) return NULL; - if (!bio->bi_css) + if (!bio->bi_blkg) return NULL; - return cgroup_get_kernfs_id(bio->bi_css->cgroup); + return cgroup_get_kernfs_id(bio_blkcg(bio)->css.cgroup); } #else static union kernfs_node_id * diff --git a/kernel/trace/preemptirq_delay_test.c b/kernel/trace/preemptirq_delay_test.c index f704390db9fc..d8765c952fab 100644 --- a/kernel/trace/preemptirq_delay_test.c +++ b/kernel/trace/preemptirq_delay_test.c @@ -5,12 +5,12 @@ * Copyright (C) 2018 Joel Fernandes (Google) <joel@joelfernandes.org> */ +#include <linux/trace_clock.h> #include <linux/delay.h> #include <linux/interrupt.h> #include <linux/irq.h> #include <linux/kernel.h> #include <linux/kthread.h> -#include <linux/ktime.h> #include <linux/module.h> #include <linux/printk.h> #include <linux/string.h> @@ -25,13 +25,13 @@ MODULE_PARM_DESC(test_mode, "Mode of the test such as preempt or irq (default ir static void busy_wait(ulong time) { - ktime_t start, end; - start = ktime_get(); + u64 start, end; + start = trace_clock_local(); do { - end = ktime_get(); + end = trace_clock_local(); if (kthread_should_stop()) break; - } while (ktime_to_ns(ktime_sub(end, start)) < (time * 1000)); + } while ((end - start) < (time * 1000)); } static int preemptirq_delay_run(void *data) diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 85f6b01431c7..d239004aaf29 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -738,16 +738,30 @@ static void free_synth_field(struct synth_field *field) kfree(field); } -static struct synth_field *parse_synth_field(char *field_type, - char *field_name) +static struct synth_field *parse_synth_field(int argc, char **argv, + int *consumed) { struct synth_field *field; + const char *prefix = NULL; + char *field_type = argv[0], *field_name; int len, ret = 0; char *array; if (field_type[0] == ';') field_type++; + if (!strcmp(field_type, "unsigned")) { + if (argc < 3) + return ERR_PTR(-EINVAL); + prefix = "unsigned "; + field_type = argv[1]; + field_name = argv[2]; + *consumed = 3; + } else { + field_name = argv[1]; + *consumed = 2; + } + len = strlen(field_name); if (field_name[len - 1] == ';') field_name[len - 1] = '\0'; @@ -760,11 +774,15 @@ static struct synth_field *parse_synth_field(char *field_type, array = strchr(field_name, '['); if (array) len += strlen(array); + if (prefix) + len += strlen(prefix); field->type = kzalloc(len, GFP_KERNEL); if (!field->type) { ret = -ENOMEM; goto free; } + if (prefix) + strcat(field->type, prefix); strcat(field->type, field_type); if (array) { strcat(field->type, array); @@ -1009,7 +1027,7 @@ static int create_synth_event(int argc, char **argv) struct synth_field *field, *fields[SYNTH_FIELDS_MAX]; struct synth_event *event = NULL; bool delete_event = false; - int i, n_fields = 0, ret = 0; + int i, consumed = 0, n_fields = 0, ret = 0; char *name; mutex_lock(&synth_event_mutex); @@ -1061,16 +1079,16 @@ static int create_synth_event(int argc, char **argv) goto err; } - field = parse_synth_field(argv[i], argv[i + 1]); + field = parse_synth_field(argc - i, &argv[i], &consumed); if (IS_ERR(field)) { ret = PTR_ERR(field); goto err; } - fields[n_fields] = field; - i++; n_fields++; + fields[n_fields++] = field; + i += consumed - 1; } - if (i < argc) { + if (i < argc && strcmp(argv[i], ";") != 0) { ret = -EINVAL; goto err; } diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c index bf2c06ef9afc..a3be42304485 100644 --- a/kernel/tracepoint.c +++ b/kernel/tracepoint.c @@ -28,8 +28,8 @@ #include <linux/sched/task.h> #include <linux/static_key.h> -extern struct tracepoint * const __start___tracepoints_ptrs[]; -extern struct tracepoint * const __stop___tracepoints_ptrs[]; +extern tracepoint_ptr_t __start___tracepoints_ptrs[]; +extern tracepoint_ptr_t __stop___tracepoints_ptrs[]; DEFINE_SRCU(tracepoint_srcu); EXPORT_SYMBOL_GPL(tracepoint_srcu); @@ -371,25 +371,17 @@ int tracepoint_probe_unregister(struct tracepoint *tp, void *probe, void *data) } EXPORT_SYMBOL_GPL(tracepoint_probe_unregister); -static void for_each_tracepoint_range(struct tracepoint * const *begin, - struct tracepoint * const *end, +static void for_each_tracepoint_range( + tracepoint_ptr_t *begin, tracepoint_ptr_t *end, void (*fct)(struct tracepoint *tp, void *priv), void *priv) { + tracepoint_ptr_t *iter; + if (!begin) return; - - if (IS_ENABLED(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS)) { - const int *iter; - - for (iter = (const int *)begin; iter < (const int *)end; iter++) - fct(offset_to_ptr(iter), priv); - } else { - struct tracepoint * const *iter; - - for (iter = begin; iter < end; iter++) - fct(*iter, priv); - } + for (iter = begin; iter < end; iter++) + fct(tracepoint_ptr_deref(iter), priv); } #ifdef CONFIG_MODULES diff --git a/lib/crc32.c b/lib/crc32.c index a6c9afafc8c8..45b1d67a1767 100644 --- a/lib/crc32.c +++ b/lib/crc32.c @@ -183,21 +183,21 @@ static inline u32 __pure crc32_le_generic(u32 crc, unsigned char const *p, } #if CRC_LE_BITS == 1 -u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len) +u32 __pure __weak crc32_le(u32 crc, unsigned char const *p, size_t len) { return crc32_le_generic(crc, p, len, NULL, CRC32_POLY_LE); } -u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len) +u32 __pure __weak __crc32c_le(u32 crc, unsigned char const *p, size_t len) { return crc32_le_generic(crc, p, len, NULL, CRC32C_POLY_LE); } #else -u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len) +u32 __pure __weak crc32_le(u32 crc, unsigned char const *p, size_t len) { return crc32_le_generic(crc, p, len, (const u32 (*)[256])crc32table_le, CRC32_POLY_LE); } -u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len) +u32 __pure __weak __crc32c_le(u32 crc, unsigned char const *p, size_t len) { return crc32_le_generic(crc, p, len, (const u32 (*)[256])crc32ctable_le, CRC32C_POLY_LE); @@ -206,6 +206,9 @@ u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len) EXPORT_SYMBOL(crc32_le); EXPORT_SYMBOL(__crc32c_le); +u32 crc32_le_base(u32, unsigned char const *, size_t) __alias(crc32_le); +u32 __crc32c_le_base(u32, unsigned char const *, size_t) __alias(__crc32c_le); + /* * This multiplies the polynomials x and y modulo the given modulus. * This follows the "little-endian" CRC convention that the lsbit diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index 9f96fa7bc000..de10b8c0bff6 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -356,11 +356,35 @@ EXPORT_SYMBOL_GPL(percpu_ref_kill_and_confirm); */ void percpu_ref_reinit(struct percpu_ref *ref) { + WARN_ON_ONCE(!percpu_ref_is_zero(ref)); + + percpu_ref_resurrect(ref); +} +EXPORT_SYMBOL_GPL(percpu_ref_reinit); + +/** + * percpu_ref_resurrect - modify a percpu refcount from dead to live + * @ref: perpcu_ref to resurrect + * + * Modify @ref so that it's in the same state as before percpu_ref_kill() was + * called. @ref must be dead but must not yet have exited. + * + * If @ref->release() frees @ref then the caller is responsible for + * guaranteeing that @ref->release() does not get called while this + * function is in progress. + * + * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while + * this function is in progress. + */ +void percpu_ref_resurrect(struct percpu_ref *ref) +{ + unsigned long __percpu *percpu_count; unsigned long flags; spin_lock_irqsave(&percpu_ref_switch_lock, flags); - WARN_ON_ONCE(!percpu_ref_is_zero(ref)); + WARN_ON_ONCE(!(ref->percpu_count_ptr & __PERCPU_REF_DEAD)); + WARN_ON_ONCE(__ref_is_percpu(ref, &percpu_count)); ref->percpu_count_ptr &= ~__PERCPU_REF_DEAD; percpu_ref_get(ref); @@ -368,4 +392,4 @@ void percpu_ref_reinit(struct percpu_ref *ref) spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); } -EXPORT_SYMBOL_GPL(percpu_ref_reinit); +EXPORT_SYMBOL_GPL(percpu_ref_resurrect); diff --git a/lib/test_ida.c b/lib/test_ida.c index 2d1637d8136b..b06880625961 100644 --- a/lib/test_ida.c +++ b/lib/test_ida.c @@ -150,10 +150,10 @@ static void ida_check_conv(struct ida *ida) IDA_BUG_ON(ida, !ida_is_empty(ida)); } +static DEFINE_IDA(ida); + static int ida_checks(void) { - DEFINE_IDA(ida); - IDA_BUG_ON(&ida, !ida_is_empty(&ida)); ida_check_alloc(&ida); ida_check_destroy(&ida); diff --git a/mm/Makefile b/mm/Makefile index 26ef77a3883b..6485d5745dd7 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -23,9 +23,9 @@ KCOV_INSTRUMENT_vmstat.o := n mmu-y := nommu.o mmu-$(CONFIG_MMU) := gup.o highmem.o memory.o mincore.o \ - mlock.o mmap.o mprotect.o mremap.o msync.o \ - page_vma_mapped.o pagewalk.o pgtable-generic.o \ - rmap.o vmalloc.o + mlock.o mmap.o mmu_gather.o mprotect.o mremap.o \ + msync.o page_vma_mapped.o pagewalk.o \ + pgtable-generic.o rmap.o vmalloc.o ifdef CONFIG_CROSS_MEMORY_ATTACH diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 58269f8ba7c4..deed97fba979 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1780,7 +1780,7 @@ static pmd_t move_soft_dirty_pmd(pmd_t pmd) bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, unsigned long new_addr, unsigned long old_end, - pmd_t *old_pmd, pmd_t *new_pmd, bool *need_flush) + pmd_t *old_pmd, pmd_t *new_pmd) { spinlock_t *old_ptl, *new_ptl; pmd_t pmd; @@ -1811,7 +1811,7 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, if (new_ptl != old_ptl) spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); - if (pmd_present(pmd) && pmd_dirty(pmd)) + if (pmd_present(pmd)) force_flush = true; VM_BUG_ON(!pmd_none(*new_pmd)); @@ -1822,12 +1822,10 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, } pmd = move_soft_dirty_pmd(pmd); set_pmd_at(mm, new_addr, new_pmd, pmd); - if (new_ptl != old_ptl) - spin_unlock(new_ptl); if (force_flush) flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE); - else - *need_flush = true; + if (new_ptl != old_ptl) + spin_unlock(new_ptl); spin_unlock(old_ptl); return true; } diff --git a/mm/memory.c b/mm/memory.c index c467102a5cbc..21a5e6e4758b 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -186,253 +186,6 @@ static void check_sync_rss_stat(struct task_struct *task) #endif /* SPLIT_RSS_COUNTING */ -#ifdef HAVE_GENERIC_MMU_GATHER - -static bool tlb_next_batch(struct mmu_gather *tlb) -{ - struct mmu_gather_batch *batch; - - batch = tlb->active; - if (batch->next) { - tlb->active = batch->next; - return true; - } - - if (tlb->batch_count == MAX_GATHER_BATCH_COUNT) - return false; - - batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0); - if (!batch) - return false; - - tlb->batch_count++; - batch->next = NULL; - batch->nr = 0; - batch->max = MAX_GATHER_BATCH; - - tlb->active->next = batch; - tlb->active = batch; - - return true; -} - -void arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, - unsigned long start, unsigned long end) -{ - tlb->mm = mm; - - /* Is it from 0 to ~0? */ - tlb->fullmm = !(start | (end+1)); - tlb->need_flush_all = 0; - tlb->local.next = NULL; - tlb->local.nr = 0; - tlb->local.max = ARRAY_SIZE(tlb->__pages); - tlb->active = &tlb->local; - tlb->batch_count = 0; - -#ifdef CONFIG_HAVE_RCU_TABLE_FREE - tlb->batch = NULL; -#endif - tlb->page_size = 0; - - __tlb_reset_range(tlb); -} - -static void tlb_flush_mmu_free(struct mmu_gather *tlb) -{ - struct mmu_gather_batch *batch; - -#ifdef CONFIG_HAVE_RCU_TABLE_FREE - tlb_table_flush(tlb); -#endif - for (batch = &tlb->local; batch && batch->nr; batch = batch->next) { - free_pages_and_swap_cache(batch->pages, batch->nr); - batch->nr = 0; - } - tlb->active = &tlb->local; -} - -void tlb_flush_mmu(struct mmu_gather *tlb) -{ - tlb_flush_mmu_tlbonly(tlb); - tlb_flush_mmu_free(tlb); -} - -/* tlb_finish_mmu - * Called at the end of the shootdown operation to free up any resources - * that were required. - */ -void arch_tlb_finish_mmu(struct mmu_gather *tlb, - unsigned long start, unsigned long end, bool force) -{ - struct mmu_gather_batch *batch, *next; - - if (force) - __tlb_adjust_range(tlb, start, end - start); - - tlb_flush_mmu(tlb); - - /* keep the page table cache within bounds */ - check_pgt_cache(); - - for (batch = tlb->local.next; batch; batch = next) { - next = batch->next; - free_pages((unsigned long)batch, 0); - } - tlb->local.next = NULL; -} - -/* __tlb_remove_page - * Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while - * handling the additional races in SMP caused by other CPUs caching valid - * mappings in their TLBs. Returns the number of free page slots left. - * When out of page slots we must call tlb_flush_mmu(). - *returns true if the caller should flush. - */ -bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size) -{ - struct mmu_gather_batch *batch; - - VM_BUG_ON(!tlb->end); - VM_WARN_ON(tlb->page_size != page_size); - - batch = tlb->active; - /* - * Add the page and check if we are full. If so - * force a flush. - */ - batch->pages[batch->nr++] = page; - if (batch->nr == batch->max) { - if (!tlb_next_batch(tlb)) - return true; - batch = tlb->active; - } - VM_BUG_ON_PAGE(batch->nr > batch->max, page); - - return false; -} - -#endif /* HAVE_GENERIC_MMU_GATHER */ - -#ifdef CONFIG_HAVE_RCU_TABLE_FREE - -/* - * See the comment near struct mmu_table_batch. - */ - -/* - * If we want tlb_remove_table() to imply TLB invalidates. - */ -static inline void tlb_table_invalidate(struct mmu_gather *tlb) -{ -#ifdef CONFIG_HAVE_RCU_TABLE_INVALIDATE - /* - * Invalidate page-table caches used by hardware walkers. Then we still - * need to RCU-sched wait while freeing the pages because software - * walkers can still be in-flight. - */ - tlb_flush_mmu_tlbonly(tlb); -#endif -} - -static void tlb_remove_table_smp_sync(void *arg) -{ - /* Simply deliver the interrupt */ -} - -static void tlb_remove_table_one(void *table) -{ - /* - * This isn't an RCU grace period and hence the page-tables cannot be - * assumed to be actually RCU-freed. - * - * It is however sufficient for software page-table walkers that rely on - * IRQ disabling. See the comment near struct mmu_table_batch. - */ - smp_call_function(tlb_remove_table_smp_sync, NULL, 1); - __tlb_remove_table(table); -} - -static void tlb_remove_table_rcu(struct rcu_head *head) -{ - struct mmu_table_batch *batch; - int i; - - batch = container_of(head, struct mmu_table_batch, rcu); - - for (i = 0; i < batch->nr; i++) - __tlb_remove_table(batch->tables[i]); - - free_page((unsigned long)batch); -} - -void tlb_table_flush(struct mmu_gather *tlb) -{ - struct mmu_table_batch **batch = &tlb->batch; - - if (*batch) { - tlb_table_invalidate(tlb); - call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu); - *batch = NULL; - } -} - -void tlb_remove_table(struct mmu_gather *tlb, void *table) -{ - struct mmu_table_batch **batch = &tlb->batch; - - if (*batch == NULL) { - *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN); - if (*batch == NULL) { - tlb_table_invalidate(tlb); - tlb_remove_table_one(table); - return; - } - (*batch)->nr = 0; - } - - (*batch)->tables[(*batch)->nr++] = table; - if ((*batch)->nr == MAX_TABLE_BATCH) - tlb_table_flush(tlb); -} - -#endif /* CONFIG_HAVE_RCU_TABLE_FREE */ - -/** - * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down - * @tlb: the mmu_gather structure to initialize - * @mm: the mm_struct of the target address space - * @start: start of the region that will be removed from the page-table - * @end: end of the region that will be removed from the page-table - * - * Called to initialize an (on-stack) mmu_gather structure for page-table - * tear-down from @mm. The @start and @end are set to 0 and -1 - * respectively when @mm is without users and we're going to destroy - * the full address space (exit/execve). - */ -void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, - unsigned long start, unsigned long end) -{ - arch_tlb_gather_mmu(tlb, mm, start, end); - inc_tlb_flush_pending(tlb->mm); -} - -void tlb_finish_mmu(struct mmu_gather *tlb, - unsigned long start, unsigned long end) -{ - /* - * If there are parallel threads are doing PTE changes on same range - * under non-exclusive lock(e.g., mmap_sem read-side) but defer TLB - * flush by batching, a thread has stable TLB entry can fail to flush - * the TLB by observing pte_none|!pte_dirty, for example so flush TLB - * forcefully if we detect parallel PTE batching threads. - */ - bool force = mm_tlb_flush_nested(tlb->mm); - - arch_tlb_finish_mmu(tlb, start, end, force); - dec_tlb_flush_pending(tlb->mm); -} - /* * Note: this doesn't free the actual pages themselves. That * has been handled earlier when unmapping all the memory regions. diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c new file mode 100644 index 000000000000..2a9fbc4a37d5 --- /dev/null +++ b/mm/mmu_gather.c @@ -0,0 +1,261 @@ +#include <linux/gfp.h> +#include <linux/highmem.h> +#include <linux/kernel.h> +#include <linux/mmdebug.h> +#include <linux/mm_types.h> +#include <linux/pagemap.h> +#include <linux/rcupdate.h> +#include <linux/smp.h> +#include <linux/swap.h> + +#include <asm/pgalloc.h> +#include <asm/tlb.h> + +#ifdef HAVE_GENERIC_MMU_GATHER + +static bool tlb_next_batch(struct mmu_gather *tlb) +{ + struct mmu_gather_batch *batch; + + batch = tlb->active; + if (batch->next) { + tlb->active = batch->next; + return true; + } + + if (tlb->batch_count == MAX_GATHER_BATCH_COUNT) + return false; + + batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0); + if (!batch) + return false; + + tlb->batch_count++; + batch->next = NULL; + batch->nr = 0; + batch->max = MAX_GATHER_BATCH; + + tlb->active->next = batch; + tlb->active = batch; + + return true; +} + +void arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + tlb->mm = mm; + + /* Is it from 0 to ~0? */ + tlb->fullmm = !(start | (end+1)); + tlb->need_flush_all = 0; + tlb->local.next = NULL; + tlb->local.nr = 0; + tlb->local.max = ARRAY_SIZE(tlb->__pages); + tlb->active = &tlb->local; + tlb->batch_count = 0; + +#ifdef CONFIG_HAVE_RCU_TABLE_FREE + tlb->batch = NULL; +#endif + tlb->page_size = 0; + + __tlb_reset_range(tlb); +} + +void tlb_flush_mmu_free(struct mmu_gather *tlb) +{ + struct mmu_gather_batch *batch; + +#ifdef CONFIG_HAVE_RCU_TABLE_FREE + tlb_table_flush(tlb); +#endif + for (batch = &tlb->local; batch && batch->nr; batch = batch->next) { + free_pages_and_swap_cache(batch->pages, batch->nr); + batch->nr = 0; + } + tlb->active = &tlb->local; +} + +void tlb_flush_mmu(struct mmu_gather *tlb) +{ + tlb_flush_mmu_tlbonly(tlb); + tlb_flush_mmu_free(tlb); +} + +/* tlb_finish_mmu + * Called at the end of the shootdown operation to free up any resources + * that were required. + */ +void arch_tlb_finish_mmu(struct mmu_gather *tlb, + unsigned long start, unsigned long end, bool force) +{ + struct mmu_gather_batch *batch, *next; + + if (force) { + __tlb_reset_range(tlb); + __tlb_adjust_range(tlb, start, end - start); + } + + tlb_flush_mmu(tlb); + + /* keep the page table cache within bounds */ + check_pgt_cache(); + + for (batch = tlb->local.next; batch; batch = next) { + next = batch->next; + free_pages((unsigned long)batch, 0); + } + tlb->local.next = NULL; +} + +/* __tlb_remove_page + * Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while + * handling the additional races in SMP caused by other CPUs caching valid + * mappings in their TLBs. Returns the number of free page slots left. + * When out of page slots we must call tlb_flush_mmu(). + *returns true if the caller should flush. + */ +bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size) +{ + struct mmu_gather_batch *batch; + + VM_BUG_ON(!tlb->end); + VM_WARN_ON(tlb->page_size != page_size); + + batch = tlb->active; + /* + * Add the page and check if we are full. If so + * force a flush. + */ + batch->pages[batch->nr++] = page; + if (batch->nr == batch->max) { + if (!tlb_next_batch(tlb)) + return true; + batch = tlb->active; + } + VM_BUG_ON_PAGE(batch->nr > batch->max, page); + + return false; +} + +#endif /* HAVE_GENERIC_MMU_GATHER */ + +#ifdef CONFIG_HAVE_RCU_TABLE_FREE + +/* + * See the comment near struct mmu_table_batch. + */ + +/* + * If we want tlb_remove_table() to imply TLB invalidates. + */ +static inline void tlb_table_invalidate(struct mmu_gather *tlb) +{ +#ifdef CONFIG_HAVE_RCU_TABLE_INVALIDATE + /* + * Invalidate page-table caches used by hardware walkers. Then we still + * need to RCU-sched wait while freeing the pages because software + * walkers can still be in-flight. + */ + tlb_flush_mmu_tlbonly(tlb); +#endif +} + +static void tlb_remove_table_smp_sync(void *arg) +{ + /* Simply deliver the interrupt */ +} + +static void tlb_remove_table_one(void *table) +{ + /* + * This isn't an RCU grace period and hence the page-tables cannot be + * assumed to be actually RCU-freed. + * + * It is however sufficient for software page-table walkers that rely on + * IRQ disabling. See the comment near struct mmu_table_batch. + */ + smp_call_function(tlb_remove_table_smp_sync, NULL, 1); + __tlb_remove_table(table); +} + +static void tlb_remove_table_rcu(struct rcu_head *head) +{ + struct mmu_table_batch *batch; + int i; + + batch = container_of(head, struct mmu_table_batch, rcu); + + for (i = 0; i < batch->nr; i++) + __tlb_remove_table(batch->tables[i]); + + free_page((unsigned long)batch); +} + +void tlb_table_flush(struct mmu_gather *tlb) +{ + struct mmu_table_batch **batch = &tlb->batch; + + if (*batch) { + tlb_table_invalidate(tlb); + call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu); + *batch = NULL; + } +} + +void tlb_remove_table(struct mmu_gather *tlb, void *table) +{ + struct mmu_table_batch **batch = &tlb->batch; + + if (*batch == NULL) { + *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN); + if (*batch == NULL) { + tlb_table_invalidate(tlb); + tlb_remove_table_one(table); + return; + } + (*batch)->nr = 0; + } + + (*batch)->tables[(*batch)->nr++] = table; + if ((*batch)->nr == MAX_TABLE_BATCH) + tlb_table_flush(tlb); +} + +#endif /* CONFIG_HAVE_RCU_TABLE_FREE */ + +/** + * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down + * @tlb: the mmu_gather structure to initialize + * @mm: the mm_struct of the target address space + * @start: start of the region that will be removed from the page-table + * @end: end of the region that will be removed from the page-table + * + * Called to initialize an (on-stack) mmu_gather structure for page-table + * tear-down from @mm. The @start and @end are set to 0 and -1 + * respectively when @mm is without users and we're going to destroy + * the full address space (exit/execve). + */ +void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + arch_tlb_gather_mmu(tlb, mm, start, end); + inc_tlb_flush_pending(tlb->mm); +} + +void tlb_finish_mmu(struct mmu_gather *tlb, + unsigned long start, unsigned long end) +{ + /* + * If there are parallel threads are doing PTE changes on same range + * under non-exclusive lock(e.g., mmap_sem read-side) but defer TLB + * flush by batching, a thread has stable TLB entry can fail to flush + * the TLB by observing pte_none|!pte_dirty, for example so flush TLB + * forcefully if we detect parallel PTE batching threads. + */ + bool force = mm_tlb_flush_nested(tlb->mm); + + arch_tlb_finish_mmu(tlb, start, end, force); + dec_tlb_flush_pending(tlb->mm); +} diff --git a/mm/mremap.c b/mm/mremap.c index 5c2e18505f75..a9617e72e6b7 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -115,7 +115,7 @@ static pte_t move_soft_dirty_pte(pte_t pte) static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, unsigned long old_addr, unsigned long old_end, struct vm_area_struct *new_vma, pmd_t *new_pmd, - unsigned long new_addr, bool need_rmap_locks, bool *need_flush) + unsigned long new_addr, bool need_rmap_locks) { struct mm_struct *mm = vma->vm_mm; pte_t *old_pte, *new_pte, pte; @@ -163,15 +163,17 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, pte = ptep_get_and_clear(mm, old_addr, old_pte); /* - * If we are remapping a dirty PTE, make sure + * If we are remapping a valid PTE, make sure * to flush TLB before we drop the PTL for the - * old PTE or we may race with page_mkclean(). + * PTE. * - * This check has to be done after we removed the - * old PTE from page tables or another thread may - * dirty it after the check and before the removal. + * NOTE! Both old and new PTL matter: the old one + * for racing with page_mkclean(), the new one to + * make sure the physical page stays valid until + * the TLB entry for the old mapping has been + * flushed. */ - if (pte_present(pte) && pte_dirty(pte)) + if (pte_present(pte)) force_flush = true; pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr); pte = move_soft_dirty_pte(pte); @@ -179,13 +181,11 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, } arch_leave_lazy_mmu_mode(); + if (force_flush) + flush_tlb_range(vma, old_end - len, old_end); if (new_ptl != old_ptl) spin_unlock(new_ptl); pte_unmap(new_pte - 1); - if (force_flush) - flush_tlb_range(vma, old_end - len, old_end); - else - *need_flush = true; pte_unmap_unlock(old_pte - 1, old_ptl); if (need_rmap_locks) drop_rmap_locks(vma); @@ -198,7 +198,6 @@ unsigned long move_page_tables(struct vm_area_struct *vma, { unsigned long extent, next, old_end; pmd_t *old_pmd, *new_pmd; - bool need_flush = false; unsigned long mmun_start; /* For mmu_notifiers */ unsigned long mmun_end; /* For mmu_notifiers */ @@ -229,8 +228,7 @@ unsigned long move_page_tables(struct vm_area_struct *vma, if (need_rmap_locks) take_rmap_locks(vma); moved = move_huge_pmd(vma, old_addr, new_addr, - old_end, old_pmd, new_pmd, - &need_flush); + old_end, old_pmd, new_pmd); if (need_rmap_locks) drop_rmap_locks(vma); if (moved) @@ -246,10 +244,8 @@ unsigned long move_page_tables(struct vm_area_struct *vma, if (extent > next - new_addr) extent = next - new_addr; move_ptes(vma, old_pmd, old_addr, old_addr + extent, new_vma, - new_pmd, new_addr, need_rmap_locks, &need_flush); + new_pmd, new_addr, need_rmap_locks); } - if (need_flush) - flush_tlb_range(vma, old_end-len, old_addr); mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end); diff --git a/mm/page_io.c b/mm/page_io.c index aafd19ec1db4..573d3663d846 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -339,7 +339,7 @@ int __swap_writepage(struct page *page, struct writeback_control *wbc, goto out; } bio->bi_opf = REQ_OP_WRITE | REQ_SWAP | wbc_to_write_flags(wbc); - bio_associate_blkcg_from_page(bio, page); + bio_associate_blkg_from_page(bio, page); count_swpout_vm_event(page); set_page_writeback(page); unlock_page(page); diff --git a/net/bpfilter/bpfilter_kern.c b/net/bpfilter/bpfilter_kern.c index b64e1649993b..94e88f510c5b 100644 --- a/net/bpfilter/bpfilter_kern.c +++ b/net/bpfilter/bpfilter_kern.c @@ -23,9 +23,11 @@ static void shutdown_umh(struct umh_info *info) if (!info->pid) return; - tsk = pid_task(find_vpid(info->pid), PIDTYPE_PID); - if (tsk) + tsk = get_pid_task(find_vpid(info->pid), PIDTYPE_PID); + if (tsk) { force_sig(SIGKILL, tsk); + put_task_struct(tsk); + } fput(info->pipe_to_umh); fput(info->pipe_from_umh); info->pid = 0; diff --git a/net/core/ethtool.c b/net/core/ethtool.c index 0762aaf8e964..aeabc4831fca 100644 --- a/net/core/ethtool.c +++ b/net/core/ethtool.c @@ -1015,6 +1015,9 @@ static noinline_for_stack int ethtool_get_rxnfc(struct net_device *dev, return -EINVAL; } + if (info.cmd != cmd) + return -EINVAL; + if (info.cmd == ETHTOOL_GRXCLSRLALL) { if (info.rule_cnt > 0) { if (info.rule_cnt <= KMALLOC_MAX_SIZE / sizeof(u32)) @@ -2469,13 +2472,17 @@ roll_back: return ret; } -static int ethtool_set_per_queue(struct net_device *dev, void __user *useraddr) +static int ethtool_set_per_queue(struct net_device *dev, + void __user *useraddr, u32 sub_cmd) { struct ethtool_per_queue_op per_queue_opt; if (copy_from_user(&per_queue_opt, useraddr, sizeof(per_queue_opt))) return -EFAULT; + if (per_queue_opt.sub_command != sub_cmd) + return -EINVAL; + switch (per_queue_opt.sub_command) { case ETHTOOL_GCOALESCE: return ethtool_get_per_queue_coalesce(dev, useraddr, &per_queue_opt); @@ -2846,7 +2853,7 @@ int dev_ethtool(struct net *net, struct ifreq *ifr) rc = ethtool_get_phy_stats(dev, useraddr); break; case ETHTOOL_PERQUEUE: - rc = ethtool_set_per_queue(dev, useraddr); + rc = ethtool_set_per_queue(dev, useraddr, sub_cmd); break; case ETHTOOL_GLINKSETTINGS: rc = ethtool_get_link_ksettings(dev, useraddr); diff --git a/net/core/neighbour.c b/net/core/neighbour.c index 91592fceeaad..4e07824eec5e 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c @@ -1148,8 +1148,7 @@ int neigh_update(struct neighbour *neigh, const u8 *lladdr, u8 new, neigh->nud_state = new; err = 0; notify = old & NUD_VALID; - if (((old & (NUD_INCOMPLETE | NUD_PROBE)) || - (flags & NEIGH_UPDATE_F_ADMIN)) && + if ((old & (NUD_INCOMPLETE | NUD_PROBE)) && (new & NUD_FAILED)) { neigh_invalidate(neigh); notify = 1; diff --git a/net/core/netpoll.c b/net/core/netpoll.c index de1d1ba92f2d..3ae899805f8b 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -312,7 +312,6 @@ void netpoll_send_skb_on_dev(struct netpoll *np, struct sk_buff *skb, /* It is up to the caller to keep npinfo alive. */ struct netpoll_info *npinfo; - rcu_read_lock_bh(); lockdep_assert_irqs_disabled(); npinfo = rcu_dereference_bh(np->dev->npinfo); @@ -357,7 +356,6 @@ void netpoll_send_skb_on_dev(struct netpoll *np, struct sk_buff *skb, skb_queue_tail(&npinfo->txq, skb); schedule_delayed_work(&npinfo->tx_work,0); } - rcu_read_unlock_bh(); } EXPORT_SYMBOL(netpoll_send_skb_on_dev); diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 428094b577fc..f817f336595d 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1846,8 +1846,9 @@ int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len) if (skb->ip_summed == CHECKSUM_COMPLETE) { int delta = skb->len - len; - skb->csum = csum_sub(skb->csum, - skb_checksum(skb, len, delta, 0)); + skb->csum = csum_block_sub(skb->csum, + skb_checksum(skb, len, delta, 0), + len); } return __pskb_trim(skb, len); } diff --git a/net/ipv4/ipmr_base.c b/net/ipv4/ipmr_base.c index 1ad9aa62a97b..eab8cd5ec2f5 100644 --- a/net/ipv4/ipmr_base.c +++ b/net/ipv4/ipmr_base.c @@ -296,8 +296,6 @@ int mr_rtm_dumproute(struct sk_buff *skb, struct netlink_callback *cb, next_entry: e++; } - e = 0; - s_e = 0; spin_lock_bh(lock); list_for_each_entry(mfc, &mrt->mfc_unres_queue, list) { diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index c63ccce6425f..4e81ff2f4588 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -4928,8 +4928,8 @@ static int in6_dump_addrs(struct inet6_dev *idev, struct sk_buff *skb, /* unicast address incl. temp addr */ list_for_each_entry(ifa, &idev->addr_list, if_list) { - if (++ip_idx < s_ip_idx) - continue; + if (ip_idx < s_ip_idx) + goto next; err = inet6_fill_ifaddr(skb, ifa, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, @@ -4938,6 +4938,8 @@ static int in6_dump_addrs(struct inet6_dev *idev, struct sk_buff *skb, if (err < 0) break; nl_dump_check_consistent(cb, nlmsg_hdr(skb)); +next: + ip_idx++; } break; } diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index a0b6932c3afd..a9d06d4dd057 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -1184,11 +1184,6 @@ route_lookup: } skb_dst_set(skb, dst); - if (encap_limit >= 0) { - init_tel_txopt(&opt, encap_limit); - ipv6_push_frag_opts(skb, &opt.ops, &proto); - } - if (hop_limit == 0) { if (skb->protocol == htons(ETH_P_IP)) hop_limit = ip_hdr(skb)->ttl; @@ -1210,6 +1205,11 @@ route_lookup: if (err) return err; + if (encap_limit >= 0) { + init_tel_txopt(&opt, encap_limit); + ipv6_push_frag_opts(skb, &opt.ops, &proto); + } + skb_push(skb, sizeof(struct ipv6hdr)); skb_reset_network_header(skb); ipv6h = ipv6_hdr(skb); diff --git a/net/ipv6/mcast.c b/net/ipv6/mcast.c index 4ae54aaca373..dbab62e3f0d7 100644 --- a/net/ipv6/mcast.c +++ b/net/ipv6/mcast.c @@ -2436,17 +2436,17 @@ static int ip6_mc_leave_src(struct sock *sk, struct ipv6_mc_socklist *iml, { int err; - /* callers have the socket lock and rtnl lock - * so no other readers or writers of iml or its sflist - */ + write_lock_bh(&iml->sflock); if (!iml->sflist) { /* any-source empty exclude case */ - return ip6_mc_del_src(idev, &iml->addr, iml->sfmode, 0, NULL, 0); + err = ip6_mc_del_src(idev, &iml->addr, iml->sfmode, 0, NULL, 0); + } else { + err = ip6_mc_del_src(idev, &iml->addr, iml->sfmode, + iml->sflist->sl_count, iml->sflist->sl_addr, 0); + sock_kfree_s(sk, iml->sflist, IP6_SFLSIZE(iml->sflist->sl_max)); + iml->sflist = NULL; } - err = ip6_mc_del_src(idev, &iml->addr, iml->sfmode, - iml->sflist->sl_count, iml->sflist->sl_addr, 0); - sock_kfree_s(sk, iml->sflist, IP6_SFLSIZE(iml->sflist->sl_max)); - iml->sflist = NULL; + write_unlock_bh(&iml->sflock); return err; } diff --git a/net/ipv6/route.c b/net/ipv6/route.c index a366c05a239d..abcb5ae77319 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -520,10 +520,11 @@ static void rt6_probe_deferred(struct work_struct *w) static void rt6_probe(struct fib6_info *rt) { - struct __rt6_probe_work *work; + struct __rt6_probe_work *work = NULL; const struct in6_addr *nh_gw; struct neighbour *neigh; struct net_device *dev; + struct inet6_dev *idev; /* * Okay, this does not seem to be appropriate @@ -539,15 +540,12 @@ static void rt6_probe(struct fib6_info *rt) nh_gw = &rt->fib6_nh.nh_gw; dev = rt->fib6_nh.nh_dev; rcu_read_lock_bh(); + idev = __in6_dev_get(dev); neigh = __ipv6_neigh_lookup_noref(dev, nh_gw); if (neigh) { - struct inet6_dev *idev; - if (neigh->nud_state & NUD_VALID) goto out; - idev = __in6_dev_get(dev); - work = NULL; write_lock(&neigh->lock); if (!(neigh->nud_state & NUD_VALID) && time_after(jiffies, @@ -557,11 +555,13 @@ static void rt6_probe(struct fib6_info *rt) __neigh_set_probe_once(neigh); } write_unlock(&neigh->lock); - } else { + } else if (time_after(jiffies, rt->last_probe + + idev->cnf.rtr_probe_interval)) { work = kmalloc(sizeof(*work), GFP_ATOMIC); } if (work) { + rt->last_probe = jiffies; INIT_WORK(&work->work, rt6_probe_deferred); work->target = *nh_gw; dev_hold(dev); diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index 28c4aa5078fc..b36694b6716e 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -766,11 +766,9 @@ static int udp6_unicast_rcv_skb(struct sock *sk, struct sk_buff *skb, ret = udpv6_queue_rcv_skb(sk, skb); - /* a return value > 0 means to resubmit the input, but - * it wants the return to be -protocol, or 0 - */ + /* a return value > 0 means to resubmit the input */ if (ret > 0) - return -ret; + return ret; return 0; } diff --git a/net/ipv6/xfrm6_policy.c b/net/ipv6/xfrm6_policy.c index ef3defaf43b9..d35bcf92969c 100644 --- a/net/ipv6/xfrm6_policy.c +++ b/net/ipv6/xfrm6_policy.c @@ -146,8 +146,8 @@ _decode_session6(struct sk_buff *skb, struct flowi *fl, int reverse) fl6->daddr = reverse ? hdr->saddr : hdr->daddr; fl6->saddr = reverse ? hdr->daddr : hdr->saddr; - while (nh + offset + 1 < skb->data || - pskb_may_pull(skb, nh + offset + 1 - skb->data)) { + while (nh + offset + sizeof(*exthdr) < skb->data || + pskb_may_pull(skb, nh + offset + sizeof(*exthdr) - skb->data)) { nh = skb_network_header(skb); exthdr = (struct ipv6_opt_hdr *)(nh + offset); diff --git a/net/llc/llc_conn.c b/net/llc/llc_conn.c index c0ac522b48a1..4ff89cb7c86f 100644 --- a/net/llc/llc_conn.c +++ b/net/llc/llc_conn.c @@ -734,6 +734,7 @@ void llc_sap_add_socket(struct llc_sap *sap, struct sock *sk) llc_sk(sk)->sap = sap; spin_lock_bh(&sap->sk_lock); + sock_set_flag(sk, SOCK_RCU_FREE); sap->sk_count++; sk_nulls_add_node_rcu(sk, laddr_hb); hlist_add_head(&llc->dev_hash_node, dev_hb); diff --git a/net/rxrpc/call_accept.c b/net/rxrpc/call_accept.c index 652e314de38e..8079aacaecac 100644 --- a/net/rxrpc/call_accept.c +++ b/net/rxrpc/call_accept.c @@ -337,7 +337,7 @@ struct rxrpc_call *rxrpc_new_incoming_call(struct rxrpc_local *local, { struct rxrpc_skb_priv *sp = rxrpc_skb(skb); struct rxrpc_connection *conn; - struct rxrpc_peer *peer; + struct rxrpc_peer *peer = NULL; struct rxrpc_call *call; _enter(""); diff --git a/net/rxrpc/local_object.c b/net/rxrpc/local_object.c index cad0691c2bb4..0906e51d3cfb 100644 --- a/net/rxrpc/local_object.c +++ b/net/rxrpc/local_object.c @@ -139,7 +139,7 @@ static int rxrpc_open_socket(struct rxrpc_local *local, struct net *net) udp_sk(usk)->gro_complete = NULL; udp_encap_enable(); -#if IS_ENABLED(CONFIG_IPV6) +#if IS_ENABLED(CONFIG_AF_RXRPC_IPV6) if (local->srx.transport.family == AF_INET6) udpv6_encap_enable(); #endif diff --git a/net/rxrpc/output.c b/net/rxrpc/output.c index e8fb8922bca8..a141ee3ab812 100644 --- a/net/rxrpc/output.c +++ b/net/rxrpc/output.c @@ -572,7 +572,8 @@ void rxrpc_reject_packets(struct rxrpc_local *local) whdr.flags ^= RXRPC_CLIENT_INITIATED; whdr.flags &= RXRPC_CLIENT_INITIATED; - ret = kernel_sendmsg(local->socket, &msg, iov, 2, size); + ret = kernel_sendmsg(local->socket, &msg, + iov, ioc, size); if (ret < 0) trace_rxrpc_tx_fail(local->debug_id, 0, ret, rxrpc_tx_point_reject); diff --git a/net/rxrpc/peer_event.c b/net/rxrpc/peer_event.c index 05b51bdbdd41..bd2fa3b7caa7 100644 --- a/net/rxrpc/peer_event.c +++ b/net/rxrpc/peer_event.c @@ -195,6 +195,7 @@ void rxrpc_error_report(struct sock *sk) rxrpc_store_error(peer, serr); rcu_read_unlock(); rxrpc_free_skb(skb, rxrpc_skb_rx_freed); + rxrpc_put_peer(peer); _leave(""); } diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 0a75cb2e5e7b..70f144ac5e1d 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -31,6 +31,8 @@ #include <net/pkt_sched.h> #include <net/pkt_cls.h> +extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1]; + /* The list of all installed classifier types */ static LIST_HEAD(tcf_proto_base); @@ -1211,7 +1213,7 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, replay: tp_created = 0; - err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, NULL, extack); + err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack); if (err < 0) return err; @@ -1360,7 +1362,7 @@ static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n, if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) return -EPERM; - err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, NULL, extack); + err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack); if (err < 0) return err; @@ -1475,7 +1477,7 @@ static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n, void *fh = NULL; int err; - err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, NULL, extack); + err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack); if (err < 0) return err; @@ -1838,7 +1840,7 @@ static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n, return -EPERM; replay: - err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, NULL, extack); + err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack); if (err < 0) return err; @@ -1949,7 +1951,8 @@ static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb) if (nlmsg_len(cb->nlh) < sizeof(*tcm)) return skb->len; - err = nlmsg_parse(cb->nlh, sizeof(*tcm), tca, TCA_MAX, NULL, NULL); + err = nlmsg_parse(cb->nlh, sizeof(*tcm), tca, TCA_MAX, rtm_tca_policy, + NULL); if (err) return err; diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 85e73f48e48f..3dc0acf54245 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -1307,10 +1307,6 @@ check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w) return 0; } -/* - * Delete/get qdisc. - */ - const struct nla_policy rtm_tca_policy[TCA_MAX + 1] = { [TCA_KIND] = { .type = NLA_STRING }, [TCA_OPTIONS] = { .type = NLA_NESTED }, @@ -1323,6 +1319,10 @@ const struct nla_policy rtm_tca_policy[TCA_MAX + 1] = { [TCA_EGRESS_BLOCK] = { .type = NLA_U32 }, }; +/* + * Delete/get qdisc. + */ + static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, struct netlink_ext_ack *extack) { @@ -2059,7 +2059,8 @@ static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb, if (tcm->tcm_parent) { q = qdisc_match_from_root(root, TC_H_MAJ(tcm->tcm_parent)); - if (q && tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0) + if (q && q != root && + tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0) return -1; return 0; } diff --git a/net/sctp/associola.c b/net/sctp/associola.c index 297d9cf960b9..a827a1f562bf 100644 --- a/net/sctp/associola.c +++ b/net/sctp/associola.c @@ -1450,7 +1450,8 @@ void sctp_assoc_sync_pmtu(struct sctp_association *asoc) /* Get the lowest pmtu of all the transports. */ list_for_each_entry(t, &asoc->peer.transport_addr_list, transports) { if (t->pmtu_pending && t->dst) { - sctp_transport_update_pmtu(t, sctp_dst_mtu(t->dst)); + sctp_transport_update_pmtu(t, + atomic_read(&t->mtu_info)); t->pmtu_pending = 0; } if (!pmtu || (t->pathmtu < pmtu)) diff --git a/net/sctp/input.c b/net/sctp/input.c index 9bbc5f92c941..5c36a99882ed 100644 --- a/net/sctp/input.c +++ b/net/sctp/input.c @@ -395,6 +395,7 @@ void sctp_icmp_frag_needed(struct sock *sk, struct sctp_association *asoc, return; if (sock_owned_by_user(sk)) { + atomic_set(&t->mtu_info, pmtu); asoc->pmtu_pending = 1; t->pmtu_pending = 1; return; diff --git a/net/sctp/output.c b/net/sctp/output.c index 7f849b01ec8e..67939ad99c01 100644 --- a/net/sctp/output.c +++ b/net/sctp/output.c @@ -120,6 +120,12 @@ void sctp_packet_config(struct sctp_packet *packet, __u32 vtag, sctp_assoc_sync_pmtu(asoc); } + if (asoc->pmtu_pending) { + if (asoc->param_flags & SPP_PMTUD_ENABLE) + sctp_assoc_sync_pmtu(asoc); + asoc->pmtu_pending = 0; + } + /* If there a is a prepend chunk stick it on the list before * any other chunks get appended. */ diff --git a/net/sctp/socket.c b/net/sctp/socket.c index f73e9d38d5ba..c1c1bda334a4 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -271,11 +271,10 @@ struct sctp_association *sctp_id2assoc(struct sock *sk, sctp_assoc_t id) spin_lock_bh(&sctp_assocs_id_lock); asoc = (struct sctp_association *)idr_find(&sctp_assocs_id, (int)id); + if (asoc && (asoc->base.sk != sk || asoc->base.dead)) + asoc = NULL; spin_unlock_bh(&sctp_assocs_id_lock); - if (!asoc || (asoc->base.sk != sk) || asoc->base.dead) - return NULL; - return asoc; } @@ -1946,8 +1945,10 @@ static int sctp_sendmsg_to_asoc(struct sctp_association *asoc, if (sp->strm_interleave) { timeo = sock_sndtimeo(sk, 0); err = sctp_wait_for_connect(asoc, &timeo); - if (err) + if (err) { + err = -ESRCH; goto err; + } } else { wait_connect = true; } @@ -7100,14 +7101,14 @@ static int sctp_getsockopt_pr_assocstatus(struct sock *sk, int len, } policy = params.sprstat_policy; - if (policy & ~SCTP_PR_SCTP_MASK) + if (!policy || (policy & ~(SCTP_PR_SCTP_MASK | SCTP_PR_SCTP_ALL))) goto out; asoc = sctp_id2assoc(sk, params.sprstat_assoc_id); if (!asoc) goto out; - if (policy == SCTP_PR_SCTP_NONE) { + if (policy & SCTP_PR_SCTP_ALL) { params.sprstat_abandoned_unsent = 0; params.sprstat_abandoned_sent = 0; for (policy = 0; policy <= SCTP_PR_INDEX(MAX); policy++) { @@ -7159,7 +7160,7 @@ static int sctp_getsockopt_pr_streamstatus(struct sock *sk, int len, } policy = params.sprstat_policy; - if (policy & ~SCTP_PR_SCTP_MASK) + if (!policy || (policy & ~(SCTP_PR_SCTP_MASK | SCTP_PR_SCTP_ALL))) goto out; asoc = sctp_id2assoc(sk, params.sprstat_assoc_id); @@ -7175,7 +7176,7 @@ static int sctp_getsockopt_pr_streamstatus(struct sock *sk, int len, goto out; } - if (policy == SCTP_PR_SCTP_NONE) { + if (policy == SCTP_PR_SCTP_ALL) { params.sprstat_abandoned_unsent = 0; params.sprstat_abandoned_sent = 0; for (policy = 0; policy <= SCTP_PR_INDEX(MAX); policy++) { diff --git a/net/socket.c b/net/socket.c index 01f3f8f32d6f..390a8ecef4bf 100644 --- a/net/socket.c +++ b/net/socket.c @@ -2875,9 +2875,14 @@ static int ethtool_ioctl(struct net *net, struct compat_ifreq __user *ifr32) copy_in_user(&rxnfc->fs.ring_cookie, &compat_rxnfc->fs.ring_cookie, (void __user *)(&rxnfc->fs.location + 1) - - (void __user *)&rxnfc->fs.ring_cookie) || - copy_in_user(&rxnfc->rule_cnt, &compat_rxnfc->rule_cnt, - sizeof(rxnfc->rule_cnt))) + (void __user *)&rxnfc->fs.ring_cookie)) + return -EFAULT; + if (ethcmd == ETHTOOL_GRXCLSRLALL) { + if (put_user(rule_cnt, &rxnfc->rule_cnt)) + return -EFAULT; + } else if (copy_in_user(&rxnfc->rule_cnt, + &compat_rxnfc->rule_cnt, + sizeof(rxnfc->rule_cnt))) return -EFAULT; } diff --git a/net/tipc/group.c b/net/tipc/group.c index e82f13cb2dc5..06fee142f09f 100644 --- a/net/tipc/group.c +++ b/net/tipc/group.c @@ -666,6 +666,7 @@ static void tipc_group_create_event(struct tipc_group *grp, struct sk_buff *skb; struct tipc_msg *hdr; + memset(&evt, 0, sizeof(evt)); evt.event = event; evt.found_lower = m->instance; evt.found_upper = m->instance; diff --git a/net/tipc/link.c b/net/tipc/link.c index f6552e4f4b43..201c3b5bc96b 100644 --- a/net/tipc/link.c +++ b/net/tipc/link.c @@ -1041,6 +1041,7 @@ static int tipc_link_retrans(struct tipc_link *l, struct tipc_link *r, if (r->last_retransm != buf_seqno(skb)) { r->last_retransm = buf_seqno(skb); r->stale_limit = jiffies + msecs_to_jiffies(r->tolerance); + r->stale_cnt = 0; } else if (++r->stale_cnt > 99 && time_after(jiffies, r->stale_limit)) { link_retransmit_failure(l, skb); if (link_is_bc_sndlink(l)) diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c index 51b4b96f89db..3cfeb9df64b0 100644 --- a/net/tipc/name_distr.c +++ b/net/tipc/name_distr.c @@ -115,7 +115,7 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ) struct sk_buff *buf; struct distr_item *item; - list_del(&publ->binding_node); + list_del_rcu(&publ->binding_node); if (publ->scope == TIPC_NODE_SCOPE) return NULL; @@ -147,7 +147,7 @@ static void named_distribute(struct net *net, struct sk_buff_head *list, ITEM_SIZE) * ITEM_SIZE; u32 msg_rem = msg_dsz; - list_for_each_entry(publ, pls, binding_node) { + list_for_each_entry_rcu(publ, pls, binding_node) { /* Prepare next buffer: */ if (!skb) { skb = named_prepare_buf(net, PUBLICATION, msg_rem, diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 4e937cd7c17d..661504042d30 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -744,6 +744,8 @@ static int xsk_create(struct net *net, struct socket *sock, int protocol, sk->sk_destruct = xsk_destruct; sk_refcnt_debug_inc(sk); + sock_set_flag(sk, SOCK_RCU_FREE); + xs = xdp_sk(sk); mutex_init(&xs->mutex); spin_lock_init(&xs->tx_completion_lock); diff --git a/net/xfrm/xfrm_interface.c b/net/xfrm/xfrm_interface.c index 31acc6f33d98..6f05e831a73e 100644 --- a/net/xfrm/xfrm_interface.c +++ b/net/xfrm/xfrm_interface.c @@ -116,6 +116,9 @@ static void xfrmi_unlink(struct xfrmi_net *xfrmn, struct xfrm_if *xi) static void xfrmi_dev_free(struct net_device *dev) { + struct xfrm_if *xi = netdev_priv(dev); + + gro_cells_destroy(&xi->gro_cells); free_percpu(dev->tstats); } diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index f094d4b3520d..119a427d9b2b 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -632,9 +632,9 @@ static void xfrm_hash_rebuild(struct work_struct *work) break; } if (newpos) - hlist_add_behind(&policy->bydst, newpos); + hlist_add_behind_rcu(&policy->bydst, newpos); else - hlist_add_head(&policy->bydst, chain); + hlist_add_head_rcu(&policy->bydst, chain); } spin_unlock_bh(&net->xfrm.xfrm_policy_lock); @@ -774,9 +774,9 @@ int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl) break; } if (newpos) - hlist_add_behind(&policy->bydst, newpos); + hlist_add_behind_rcu(&policy->bydst, newpos); else - hlist_add_head(&policy->bydst, chain); + hlist_add_head_rcu(&policy->bydst, chain); __xfrm_policy_link(policy, dir); /* After previous checking, family can either be AF_INET or AF_INET6 */ diff --git a/sound/hda/hdac_regmap.c b/sound/hda/hdac_regmap.c index 419e285e0226..996dbc850224 100644 --- a/sound/hda/hdac_regmap.c +++ b/sound/hda/hdac_regmap.c @@ -359,7 +359,8 @@ static const struct regmap_config hda_regmap_cfg = { .cache_type = REGCACHE_RBTREE, .reg_read = hda_reg_read, .reg_write = hda_reg_write, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; /** diff --git a/sound/soc/codecs/cs35l33.c b/sound/soc/codecs/cs35l33.c index 668cd3754209..e9b7f72d880b 100644 --- a/sound/soc/codecs/cs35l33.c +++ b/sound/soc/codecs/cs35l33.c @@ -857,7 +857,8 @@ static const struct regmap_config cs35l33_regmap = { .readable_reg = cs35l33_readable_register, .writeable_reg = cs35l33_writeable_register, .cache_type = REGCACHE_RBTREE, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; static int __maybe_unused cs35l33_runtime_resume(struct device *dev) diff --git a/sound/soc/codecs/cs35l35.c b/sound/soc/codecs/cs35l35.c index bd6226bde45f..9f4a59871cee 100644 --- a/sound/soc/codecs/cs35l35.c +++ b/sound/soc/codecs/cs35l35.c @@ -1105,7 +1105,8 @@ static struct regmap_config cs35l35_regmap = { .readable_reg = cs35l35_readable_register, .precious_reg = cs35l35_precious_register, .cache_type = REGCACHE_RBTREE, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; static irqreturn_t cs35l35_irq(int irq, void *data) diff --git a/sound/soc/codecs/cs43130.c b/sound/soc/codecs/cs43130.c index 80dc42197154..3f7b255587e6 100644 --- a/sound/soc/codecs/cs43130.c +++ b/sound/soc/codecs/cs43130.c @@ -2362,7 +2362,9 @@ static const struct regmap_config cs43130_regmap = { .precious_reg = cs43130_precious_register, .volatile_reg = cs43130_volatile_register, .cache_type = REGCACHE_RBTREE, - .use_single_rw = true, /* needed for regcache_sync */ + /* needed for regcache_sync */ + .use_single_read = true, + .use_single_write = true, }; static u16 const cs43130_dc_threshold[CS43130_DC_THRESHOLD] = { diff --git a/sound/soc/codecs/es8328.c b/sound/soc/codecs/es8328.c index e9fc2fd97d2f..4b5827dc23aa 100644 --- a/sound/soc/codecs/es8328.c +++ b/sound/soc/codecs/es8328.c @@ -824,7 +824,8 @@ const struct regmap_config es8328_regmap_config = { .val_bits = 8, .max_register = ES8328_REG_MAX, .cache_type = REGCACHE_RBTREE, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; EXPORT_SYMBOL_GPL(es8328_regmap_config); diff --git a/sound/soc/codecs/rt1305.c b/sound/soc/codecs/rt1305.c index c4452efc7970..c2c8a68cec97 100644 --- a/sound/soc/codecs/rt1305.c +++ b/sound/soc/codecs/rt1305.c @@ -963,7 +963,8 @@ static const struct regmap_config rt1305_regmap = { .num_reg_defaults = ARRAY_SIZE(rt1305_reg), .ranges = rt1305_ranges, .num_ranges = ARRAY_SIZE(rt1305_ranges), - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; #if defined(CONFIG_OF) diff --git a/sound/soc/codecs/rt5514.c b/sound/soc/codecs/rt5514.c index 32fe76c3134a..a67de68b6da6 100644 --- a/sound/soc/codecs/rt5514.c +++ b/sound/soc/codecs/rt5514.c @@ -1201,7 +1201,8 @@ static const struct regmap_config rt5514_regmap = { .cache_type = REGCACHE_RBTREE, .reg_defaults = rt5514_reg, .num_reg_defaults = ARRAY_SIZE(rt5514_reg), - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; static const struct i2c_device_id rt5514_i2c_id[] = { diff --git a/sound/soc/codecs/rt5616.c b/sound/soc/codecs/rt5616.c index 3dc795f444ce..36a9f1c56c8d 100644 --- a/sound/soc/codecs/rt5616.c +++ b/sound/soc/codecs/rt5616.c @@ -1313,7 +1313,8 @@ static const struct snd_soc_component_driver soc_component_dev_rt5616 = { static const struct regmap_config rt5616_regmap = { .reg_bits = 8, .val_bits = 16, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, .max_register = RT5616_DEVICE_ID + 1 + (ARRAY_SIZE(rt5616_ranges) * RT5616_PR_SPACING), .volatile_reg = rt5616_volatile_register, diff --git a/sound/soc/codecs/rt5640.c b/sound/soc/codecs/rt5640.c index 27770143ae8f..fc530481a6e4 100644 --- a/sound/soc/codecs/rt5640.c +++ b/sound/soc/codecs/rt5640.c @@ -2704,7 +2704,8 @@ static const struct snd_soc_component_driver soc_component_dev_rt5640 = { static const struct regmap_config rt5640_regmap = { .reg_bits = 8, .val_bits = 16, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, .max_register = RT5640_VENDOR_ID2 + 1 + (ARRAY_SIZE(rt5640_ranges) * RT5640_PR_SPACING), diff --git a/sound/soc/codecs/rt5645.c b/sound/soc/codecs/rt5645.c index 1dc70f452c1b..be674688dc40 100644 --- a/sound/soc/codecs/rt5645.c +++ b/sound/soc/codecs/rt5645.c @@ -3559,7 +3559,8 @@ static const struct snd_soc_component_driver soc_component_dev_rt5645 = { static const struct regmap_config rt5645_regmap = { .reg_bits = 8, .val_bits = 16, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, .max_register = RT5645_VENDOR_ID2 + 1 + (ARRAY_SIZE(rt5645_ranges) * RT5645_PR_SPACING), .volatile_reg = rt5645_volatile_register, @@ -3575,7 +3576,8 @@ static const struct regmap_config rt5645_regmap = { static const struct regmap_config rt5650_regmap = { .reg_bits = 8, .val_bits = 16, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, .max_register = RT5645_VENDOR_ID2 + 1 + (ARRAY_SIZE(rt5645_ranges) * RT5645_PR_SPACING), .volatile_reg = rt5645_volatile_register, @@ -3592,7 +3594,8 @@ static const struct regmap_config temp_regmap = { .name="nocache", .reg_bits = 8, .val_bits = 16, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, .max_register = RT5645_VENDOR_ID2 + 1, .cache_type = REGCACHE_NONE, }; diff --git a/sound/soc/codecs/rt5651.c b/sound/soc/codecs/rt5651.c index 985852fd9723..5bcedbc7eb4a 100644 --- a/sound/soc/codecs/rt5651.c +++ b/sound/soc/codecs/rt5651.c @@ -2124,7 +2124,8 @@ static const struct regmap_config rt5651_regmap = { .num_reg_defaults = ARRAY_SIZE(rt5651_reg), .ranges = rt5651_ranges, .num_ranges = ARRAY_SIZE(rt5651_ranges), - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; #if defined(CONFIG_OF) diff --git a/sound/soc/codecs/rt5660.c b/sound/soc/codecs/rt5660.c index 20a755137e63..27f7445b2432 100644 --- a/sound/soc/codecs/rt5660.c +++ b/sound/soc/codecs/rt5660.c @@ -1217,7 +1217,8 @@ static const struct snd_soc_component_driver soc_component_dev_rt5660 = { static const struct regmap_config rt5660_regmap = { .reg_bits = 8, .val_bits = 16, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, .max_register = RT5660_VENDOR_ID2 + 1 + (ARRAY_SIZE(rt5660_ranges) * RT5660_PR_SPACING), diff --git a/sound/soc/codecs/rt5663.c b/sound/soc/codecs/rt5663.c index 9bd24ad42240..70441661ea4a 100644 --- a/sound/soc/codecs/rt5663.c +++ b/sound/soc/codecs/rt5663.c @@ -3252,7 +3252,8 @@ static const struct snd_soc_component_driver soc_component_dev_rt5663 = { static const struct regmap_config rt5663_v2_regmap = { .reg_bits = 16, .val_bits = 16, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, .max_register = 0x07fa, .volatile_reg = rt5663_v2_volatile_register, .readable_reg = rt5663_v2_readable_register, @@ -3264,7 +3265,8 @@ static const struct regmap_config rt5663_v2_regmap = { static const struct regmap_config rt5663_regmap = { .reg_bits = 16, .val_bits = 16, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, .max_register = 0x03f3, .volatile_reg = rt5663_volatile_register, .readable_reg = rt5663_readable_register, @@ -3277,7 +3279,8 @@ static const struct regmap_config temp_regmap = { .name = "nocache", .reg_bits = 16, .val_bits = 16, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, .max_register = 0x03f3, .cache_type = REGCACHE_NONE, }; diff --git a/sound/soc/codecs/rt5665.c b/sound/soc/codecs/rt5665.c index 6ba99f5ed3f4..f2ad3a4c3b7f 100644 --- a/sound/soc/codecs/rt5665.c +++ b/sound/soc/codecs/rt5665.c @@ -4633,7 +4633,8 @@ static const struct regmap_config rt5665_regmap = { .cache_type = REGCACHE_RBTREE, .reg_defaults = rt5665_reg, .num_reg_defaults = ARRAY_SIZE(rt5665_reg), - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; static const struct i2c_device_id rt5665_i2c_id[] = { diff --git a/sound/soc/codecs/rt5668.c b/sound/soc/codecs/rt5668.c index 3c19d03f2446..3f6046a66b56 100644 --- a/sound/soc/codecs/rt5668.c +++ b/sound/soc/codecs/rt5668.c @@ -2375,7 +2375,8 @@ static const struct regmap_config rt5668_regmap = { .cache_type = REGCACHE_RBTREE, .reg_defaults = rt5668_reg, .num_reg_defaults = ARRAY_SIZE(rt5668_reg), - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; static const struct i2c_device_id rt5668_i2c_id[] = { diff --git a/sound/soc/codecs/rt5670.c b/sound/soc/codecs/rt5670.c index 732ef928b25d..f0f8debc2829 100644 --- a/sound/soc/codecs/rt5670.c +++ b/sound/soc/codecs/rt5670.c @@ -2814,7 +2814,8 @@ static const struct snd_soc_component_driver soc_component_dev_rt5670 = { static const struct regmap_config rt5670_regmap = { .reg_bits = 8, .val_bits = 16, - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, .max_register = RT5670_VENDOR_ID2 + 1 + (ARRAY_SIZE(rt5670_ranges) * RT5670_PR_SPACING), .volatile_reg = rt5670_volatile_register, diff --git a/sound/soc/codecs/rt5682.c b/sound/soc/codecs/rt5682.c index afe7d5b19313..8a3052874c29 100644 --- a/sound/soc/codecs/rt5682.c +++ b/sound/soc/codecs/rt5682.c @@ -2419,7 +2419,8 @@ static const struct regmap_config rt5682_regmap = { .cache_type = REGCACHE_RBTREE, .reg_defaults = rt5682_reg, .num_reg_defaults = ARRAY_SIZE(rt5682_reg), - .use_single_rw = true, + .use_single_read = true, + .use_single_write = true, }; static const struct i2c_device_id rt5682_i2c_id[] = { diff --git a/tools/arch/x86/include/uapi/asm/kvm.h b/tools/arch/x86/include/uapi/asm/kvm.h index 86299efa804a..fd23d5778ea1 100644 --- a/tools/arch/x86/include/uapi/asm/kvm.h +++ b/tools/arch/x86/include/uapi/asm/kvm.h @@ -377,6 +377,7 @@ struct kvm_sync_regs { #define KVM_X86_QUIRK_LINT0_REENABLED (1 << 0) #define KVM_X86_QUIRK_CD_NW_CLEARED (1 << 1) +#define KVM_X86_QUIRK_LAPIC_MMIO_HOLE (1 << 2) #define KVM_STATE_NESTED_GUEST_MODE 0x00000001 #define KVM_STATE_NESTED_RUN_PENDING 0x00000002 diff --git a/tools/include/uapi/linux/kvm.h b/tools/include/uapi/linux/kvm.h index 07548de5c988..251be353f950 100644 --- a/tools/include/uapi/linux/kvm.h +++ b/tools/include/uapi/linux/kvm.h @@ -952,6 +952,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_S390_HPAGE_1M 156 #define KVM_CAP_NESTED_STATE 157 #define KVM_CAP_ARM_INJECT_SERROR_ESR 158 +#define KVM_CAP_MSR_PLATFORM_INFO 159 #ifdef KVM_CAP_IRQ_ROUTING diff --git a/tools/lib/api/fs/tracing_path.c b/tools/lib/api/fs/tracing_path.c index 120037496f77..5afb11b30fca 100644 --- a/tools/lib/api/fs/tracing_path.c +++ b/tools/lib/api/fs/tracing_path.c @@ -36,7 +36,7 @@ static const char *tracing_path_tracefs_mount(void) __tracing_path_set("", mnt); - return mnt; + return tracing_path; } static const char *tracing_path_debugfs_mount(void) @@ -49,7 +49,7 @@ static const char *tracing_path_debugfs_mount(void) __tracing_path_set("tracing/", mnt); - return mnt; + return tracing_path; } const char *tracing_path_mount(void) diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index f6d1a03c7523..e30d20fb482d 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -833,7 +833,7 @@ ifndef NO_JVMTI JDIR=$(shell /usr/sbin/update-java-alternatives -l | head -1 | awk '{print $$3}') else ifneq (,$(wildcard /usr/sbin/alternatives)) - JDIR=$(shell alternatives --display java | tail -1 | cut -d' ' -f 5 | sed 's%/jre/bin/java.%%g') + JDIR=$(shell /usr/sbin/alternatives --display java | tail -1 | cut -d' ' -f 5 | sed 's%/jre/bin/java.%%g') endif endif ifndef JDIR diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf index 5224ade3d5af..0be411695379 100644 --- a/tools/perf/Makefile.perf +++ b/tools/perf/Makefile.perf @@ -635,7 +635,7 @@ $(LIBPERF_IN): prepare FORCE $(LIB_FILE): $(LIBPERF_IN) $(QUIET_AR)$(RM) $@ && $(AR) rcs $@ $(LIBPERF_IN) $(LIB_OBJS) -LIBTRACEEVENT_FLAGS += plugin_dir=$(plugindir_SQ) +LIBTRACEEVENT_FLAGS += plugin_dir=$(plugindir_SQ) 'EXTRA_CFLAGS=$(EXTRA_CFLAGS)' 'LDFLAGS=$(LDFLAGS)' $(LIBTRACEEVENT): FORCE $(Q)$(MAKE) -C $(TRACE_EVENT_DIR) $(LIBTRACEEVENT_FLAGS) O=$(OUTPUT) $(OUTPUT)libtraceevent.a diff --git a/tools/perf/builtin-report.c b/tools/perf/builtin-report.c index 76e12bcd1765..b2188e623e22 100644 --- a/tools/perf/builtin-report.c +++ b/tools/perf/builtin-report.c @@ -981,6 +981,7 @@ int cmd_report(int argc, const char **argv) .id_index = perf_event__process_id_index, .auxtrace_info = perf_event__process_auxtrace_info, .auxtrace = perf_event__process_auxtrace, + .event_update = perf_event__process_event_update, .feature = process_feature_event, .ordered_events = true, .ordering_requires_timestamps = true, diff --git a/tools/perf/pmu-events/arch/x86/ivytown/uncore-power.json b/tools/perf/pmu-events/arch/x86/ivytown/uncore-power.json index d40498f2cb1e..635c09fda1d9 100644 --- a/tools/perf/pmu-events/arch/x86/ivytown/uncore-power.json +++ b/tools/perf/pmu-events/arch/x86/ivytown/uncore-power.json @@ -188,7 +188,7 @@ "Counter": "0,1,2,3", "EventCode": "0xb", "EventName": "UNC_P_FREQ_GE_1200MHZ_CYCLES", - "Filter": "filter_band0=1200", + "Filter": "filter_band0=12", "MetricExpr": "(UNC_P_FREQ_GE_1200MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_1200mhz_cycles %", "PerPkg": "1", @@ -199,7 +199,7 @@ "Counter": "0,1,2,3", "EventCode": "0xc", "EventName": "UNC_P_FREQ_GE_2000MHZ_CYCLES", - "Filter": "filter_band1=2000", + "Filter": "filter_band1=20", "MetricExpr": "(UNC_P_FREQ_GE_2000MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_2000mhz_cycles %", "PerPkg": "1", @@ -210,7 +210,7 @@ "Counter": "0,1,2,3", "EventCode": "0xd", "EventName": "UNC_P_FREQ_GE_3000MHZ_CYCLES", - "Filter": "filter_band2=3000", + "Filter": "filter_band2=30", "MetricExpr": "(UNC_P_FREQ_GE_3000MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_3000mhz_cycles %", "PerPkg": "1", @@ -221,7 +221,7 @@ "Counter": "0,1,2,3", "EventCode": "0xe", "EventName": "UNC_P_FREQ_GE_4000MHZ_CYCLES", - "Filter": "filter_band3=4000", + "Filter": "filter_band3=40", "MetricExpr": "(UNC_P_FREQ_GE_4000MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_4000mhz_cycles %", "PerPkg": "1", @@ -232,7 +232,7 @@ "Counter": "0,1,2,3", "EventCode": "0xb", "EventName": "UNC_P_FREQ_GE_1200MHZ_TRANSITIONS", - "Filter": "edge=1,filter_band0=1200", + "Filter": "edge=1,filter_band0=12", "MetricExpr": "(UNC_P_FREQ_GE_1200MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_1200mhz_cycles %", "PerPkg": "1", @@ -243,7 +243,7 @@ "Counter": "0,1,2,3", "EventCode": "0xc", "EventName": "UNC_P_FREQ_GE_2000MHZ_TRANSITIONS", - "Filter": "edge=1,filter_band1=2000", + "Filter": "edge=1,filter_band1=20", "MetricExpr": "(UNC_P_FREQ_GE_2000MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_2000mhz_cycles %", "PerPkg": "1", @@ -254,7 +254,7 @@ "Counter": "0,1,2,3", "EventCode": "0xd", "EventName": "UNC_P_FREQ_GE_3000MHZ_TRANSITIONS", - "Filter": "edge=1,filter_band2=4000", + "Filter": "edge=1,filter_band2=30", "MetricExpr": "(UNC_P_FREQ_GE_3000MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_3000mhz_cycles %", "PerPkg": "1", @@ -265,7 +265,7 @@ "Counter": "0,1,2,3", "EventCode": "0xe", "EventName": "UNC_P_FREQ_GE_4000MHZ_TRANSITIONS", - "Filter": "edge=1,filter_band3=4000", + "Filter": "edge=1,filter_band3=40", "MetricExpr": "(UNC_P_FREQ_GE_4000MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_4000mhz_cycles %", "PerPkg": "1", diff --git a/tools/perf/pmu-events/arch/x86/jaketown/uncore-power.json b/tools/perf/pmu-events/arch/x86/jaketown/uncore-power.json index 16034bfd06dd..8755693d86c6 100644 --- a/tools/perf/pmu-events/arch/x86/jaketown/uncore-power.json +++ b/tools/perf/pmu-events/arch/x86/jaketown/uncore-power.json @@ -187,7 +187,7 @@ "Counter": "0,1,2,3", "EventCode": "0xb", "EventName": "UNC_P_FREQ_GE_1200MHZ_CYCLES", - "Filter": "filter_band0=1200", + "Filter": "filter_band0=12", "MetricExpr": "(UNC_P_FREQ_GE_1200MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_1200mhz_cycles %", "PerPkg": "1", @@ -198,7 +198,7 @@ "Counter": "0,1,2,3", "EventCode": "0xc", "EventName": "UNC_P_FREQ_GE_2000MHZ_CYCLES", - "Filter": "filter_band1=2000", + "Filter": "filter_band1=20", "MetricExpr": "(UNC_P_FREQ_GE_2000MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_2000mhz_cycles %", "PerPkg": "1", @@ -209,7 +209,7 @@ "Counter": "0,1,2,3", "EventCode": "0xd", "EventName": "UNC_P_FREQ_GE_3000MHZ_CYCLES", - "Filter": "filter_band2=3000", + "Filter": "filter_band2=30", "MetricExpr": "(UNC_P_FREQ_GE_3000MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_3000mhz_cycles %", "PerPkg": "1", @@ -220,7 +220,7 @@ "Counter": "0,1,2,3", "EventCode": "0xe", "EventName": "UNC_P_FREQ_GE_4000MHZ_CYCLES", - "Filter": "filter_band3=4000", + "Filter": "filter_band3=40", "MetricExpr": "(UNC_P_FREQ_GE_4000MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_4000mhz_cycles %", "PerPkg": "1", @@ -231,7 +231,7 @@ "Counter": "0,1,2,3", "EventCode": "0xb", "EventName": "UNC_P_FREQ_GE_1200MHZ_TRANSITIONS", - "Filter": "edge=1,filter_band0=1200", + "Filter": "edge=1,filter_band0=12", "MetricExpr": "(UNC_P_FREQ_GE_1200MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_1200mhz_cycles %", "PerPkg": "1", @@ -242,7 +242,7 @@ "Counter": "0,1,2,3", "EventCode": "0xc", "EventName": "UNC_P_FREQ_GE_2000MHZ_TRANSITIONS", - "Filter": "edge=1,filter_band1=2000", + "Filter": "edge=1,filter_band1=20", "MetricExpr": "(UNC_P_FREQ_GE_2000MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_2000mhz_cycles %", "PerPkg": "1", @@ -253,7 +253,7 @@ "Counter": "0,1,2,3", "EventCode": "0xd", "EventName": "UNC_P_FREQ_GE_3000MHZ_TRANSITIONS", - "Filter": "edge=1,filter_band2=4000", + "Filter": "edge=1,filter_band2=30", "MetricExpr": "(UNC_P_FREQ_GE_3000MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_3000mhz_cycles %", "PerPkg": "1", @@ -264,7 +264,7 @@ "Counter": "0,1,2,3", "EventCode": "0xe", "EventName": "UNC_P_FREQ_GE_4000MHZ_TRANSITIONS", - "Filter": "edge=1,filter_band3=4000", + "Filter": "edge=1,filter_band3=40", "MetricExpr": "(UNC_P_FREQ_GE_4000MHZ_CYCLES / UNC_P_CLOCKTICKS) * 100.", "MetricName": "freq_ge_4000mhz_cycles %", "PerPkg": "1", diff --git a/tools/perf/util/event.c b/tools/perf/util/event.c index 0cd42150f712..bc646185f8d9 100644 --- a/tools/perf/util/event.c +++ b/tools/perf/util/event.c @@ -1081,6 +1081,7 @@ void *cpu_map_data__alloc(struct cpu_map *map, size_t *size, u16 *type, int *max } *size += sizeof(struct cpu_map_data); + *size = PERF_ALIGN(*size, sizeof(u64)); return zalloc(*size); } @@ -1560,26 +1561,9 @@ struct map *thread__find_map(struct thread *thread, u8 cpumode, u64 addr, return NULL; } -try_again: + al->map = map_groups__find(mg, al->addr); - if (al->map == NULL) { - /* - * If this is outside of all known maps, and is a negative - * address, try to look it up in the kernel dso, as it might be - * a vsyscall or vdso (which executes in user-mode). - * - * XXX This is nasty, we should have a symbol list in the - * "[vdso]" dso, but for now lets use the old trick of looking - * in the whole kernel symbol list. - */ - if (cpumode == PERF_RECORD_MISC_USER && machine && - mg != &machine->kmaps && - machine__kernel_ip(machine, al->addr)) { - mg = &machine->kmaps; - load_map = true; - goto try_again; - } - } else { + if (al->map != NULL) { /* * Kernel maps might be changed when loading symbols so loading * must be done prior to using kernel maps. diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index 1a61628a1c12..e596ae358c4d 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -1089,6 +1089,9 @@ void perf_evsel__config(struct perf_evsel *evsel, struct record_opts *opts, attr->exclude_user = 1; } + if (evsel->own_cpus) + evsel->attr.read_format |= PERF_FORMAT_ID; + /* * Apply event specific term settings, * it overloads any global configuration. diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index afd68524ffa9..7799788f662f 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -930,13 +930,14 @@ static void pmu_format_value(unsigned long *format, __u64 value, __u64 *v, static __u64 pmu_format_max_value(const unsigned long *format) { - __u64 w = 0; - int fbit; - - for_each_set_bit(fbit, format, PERF_PMU_FORMAT_BITS) - w |= (1ULL << fbit); + int w; - return w; + w = bitmap_weight(format, PERF_PMU_FORMAT_BITS); + if (!w) + return 0; + if (w < 64) + return (1ULL << w) - 1; + return -1; } /* diff --git a/tools/perf/util/srcline.c b/tools/perf/util/srcline.c index 09d6746e6ec8..e767c4a9d4d2 100644 --- a/tools/perf/util/srcline.c +++ b/tools/perf/util/srcline.c @@ -85,6 +85,9 @@ static struct symbol *new_inline_sym(struct dso *dso, struct symbol *inline_sym; char *demangled = NULL; + if (!funcname) + funcname = "??"; + if (dso) { demangled = dso__demangle_sym(dso, 0, funcname); if (demangled) diff --git a/tools/spi/spidev_test.c b/tools/spi/spidev_test.c index 8c590cd1171a..4c12e6aea5d5 100644 --- a/tools/spi/spidev_test.c +++ b/tools/spi/spidev_test.c @@ -73,12 +73,12 @@ static void hex_dump(const void *src, size_t length, size_t line_size, while (i++ % line_size) printf("__ "); } - printf(" | "); /* right close */ + printf(" |"); while (line < address) { c = *line++; - printf("%c", (c < 33 || c == 255) ? 0x2E : c); + printf("%c", (c < 32 || c > 126) ? '.' : c); } - printf("\n"); + printf("|\n"); if (length > 0) printf("%s | ", prefix); } diff --git a/tools/testing/selftests/drivers/usb/usbip/usbip_test.sh b/tools/testing/selftests/drivers/usb/usbip/usbip_test.sh index a72df93cf1f8..128f0ab24307 100755 --- a/tools/testing/selftests/drivers/usb/usbip/usbip_test.sh +++ b/tools/testing/selftests/drivers/usb/usbip/usbip_test.sh @@ -141,6 +141,10 @@ echo "Import devices from localhost - should work" src/usbip attach -r localhost -b $busid; echo "==============================================================" +# Wait for sysfs file to be updated. Without this sleep, usbip port +# shows no imported devices. +sleep 3; + echo "List imported devices - expect to see imported devices"; src/usbip port; echo "==============================================================" diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-synthetic-event-syntax.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-synthetic-event-syntax.tc new file mode 100644 index 000000000000..88e6c3f43006 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-synthetic-event-syntax.tc @@ -0,0 +1,80 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: event trigger - test synthetic_events syntax parser + +do_reset() { + reset_trigger + echo > set_event + clear_trace +} + +fail() { #msg + do_reset + echo $1 + exit_fail +} + +if [ ! -f set_event ]; then + echo "event tracing is not supported" + exit_unsupported +fi + +if [ ! -f synthetic_events ]; then + echo "synthetic event is not supported" + exit_unsupported +fi + +reset_tracer +do_reset + +echo "Test synthetic_events syntax parser" + +echo > synthetic_events + +# synthetic event must have a field +! echo "myevent" >> synthetic_events +echo "myevent u64 var1" >> synthetic_events + +# synthetic event must be found in synthetic_events +grep "myevent[[:space:]]u64 var1" synthetic_events + +# it is not possible to add same name event +! echo "myevent u64 var2" >> synthetic_events + +# Non-append open will cleanup all events and add new one +echo "myevent u64 var2" > synthetic_events + +# multiple fields with different spaces +echo "myevent u64 var1; u64 var2;" > synthetic_events +grep "myevent[[:space:]]u64 var1; u64 var2" synthetic_events +echo "myevent u64 var1 ; u64 var2 ;" > synthetic_events +grep "myevent[[:space:]]u64 var1; u64 var2" synthetic_events +echo "myevent u64 var1 ;u64 var2" > synthetic_events +grep "myevent[[:space:]]u64 var1; u64 var2" synthetic_events + +# test field types +echo "myevent u32 var" > synthetic_events +echo "myevent u16 var" > synthetic_events +echo "myevent u8 var" > synthetic_events +echo "myevent s64 var" > synthetic_events +echo "myevent s32 var" > synthetic_events +echo "myevent s16 var" > synthetic_events +echo "myevent s8 var" > synthetic_events + +echo "myevent char var" > synthetic_events +echo "myevent int var" > synthetic_events +echo "myevent long var" > synthetic_events +echo "myevent pid_t var" > synthetic_events + +echo "myevent unsigned char var" > synthetic_events +echo "myevent unsigned int var" > synthetic_events +echo "myevent unsigned long var" > synthetic_events +grep "myevent[[:space:]]unsigned long var" synthetic_events + +# test string type +echo "myevent char var[10]" > synthetic_events +grep "myevent[[:space:]]char\[10\] var" synthetic_events + +do_reset + +exit 0 diff --git a/tools/testing/selftests/net/reuseport_bpf.c b/tools/testing/selftests/net/reuseport_bpf.c index cad14cd0ea92..b5277106df1f 100644 --- a/tools/testing/selftests/net/reuseport_bpf.c +++ b/tools/testing/selftests/net/reuseport_bpf.c @@ -437,14 +437,19 @@ void enable_fastopen(void) } } -static struct rlimit rlim_old, rlim_new; +static struct rlimit rlim_old; static __attribute__((constructor)) void main_ctor(void) { getrlimit(RLIMIT_MEMLOCK, &rlim_old); - rlim_new.rlim_cur = rlim_old.rlim_cur + (1UL << 20); - rlim_new.rlim_max = rlim_old.rlim_max + (1UL << 20); - setrlimit(RLIMIT_MEMLOCK, &rlim_new); + + if (rlim_old.rlim_cur != RLIM_INFINITY) { + struct rlimit rlim_new; + + rlim_new.rlim_cur = rlim_old.rlim_cur + (1UL << 20); + rlim_new.rlim_max = rlim_old.rlim_max + (1UL << 20); + setrlimit(RLIMIT_MEMLOCK, &rlim_new); + } } static __attribute__((destructor)) void main_dtor(void) diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c index c92053bc3f96..150c8a69cdaf 100644 --- a/virt/kvm/arm/arm.c +++ b/virt/kvm/arm/arm.c @@ -496,7 +496,7 @@ static bool need_new_vmid_gen(struct kvm *kvm) static void update_vttbr(struct kvm *kvm) { phys_addr_t pgd_phys; - u64 vmid; + u64 vmid, cnp = kvm_cpu_has_cnp() ? VTTBR_CNP_BIT : 0; bool new_gen; read_lock(&kvm_vmid_lock); @@ -546,7 +546,7 @@ static void update_vttbr(struct kvm *kvm) pgd_phys = virt_to_phys(kvm->arch.pgd); BUG_ON(pgd_phys & ~VTTBR_BADDR_MASK); vmid = ((u64)(kvm->arch.vmid) << VTTBR_VMID_SHIFT) & VTTBR_VMID_MASK(kvm_vmid_bits); - kvm->arch.vttbr = kvm_phys_to_vttbr(pgd_phys) | vmid; + kvm->arch.vttbr = kvm_phys_to_vttbr(pgd_phys) | vmid | cnp; write_unlock(&kvm_vmid_lock); } |