/* * Driver for IBM Power 842 compression accelerator * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Copyright (C) IBM Corporation, 2012 * * Authors: Robert Jennings * Seth Jennings */ #include #include #include #include #include #include #include #include "nx_csbcpb.h" /* struct nx_csbcpb */ #define MODULE_NAME "nx-compress" MODULE_LICENSE("GPL"); MODULE_AUTHOR("Robert Jennings "); MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors"); #define SHIFT_4K 12 #define SHIFT_64K 16 #define SIZE_4K (1UL << SHIFT_4K) #define SIZE_64K (1UL << SHIFT_64K) /* IO buffer must be 128 byte aligned */ #define IO_BUFFER_ALIGN 128 struct nx842_header { int blocks_nr; /* number of compressed blocks */ int offset; /* offset of the first block (from beginning of header) */ int sizes[0]; /* size of compressed blocks */ }; static inline int nx842_header_size(const struct nx842_header *hdr) { return sizeof(struct nx842_header) + hdr->blocks_nr * sizeof(hdr->sizes[0]); } /* Macros for fields within nx_csbcpb */ /* Check the valid bit within the csbcpb valid field */ #define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7)) /* CE macros operate on the completion_extension field bits in the csbcpb. * CE0 0=full completion, 1=partial completion * CE1 0=CE0 indicates completion, 1=termination (output may be modified) * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */ #define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7)) #define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6)) #define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5)) /* The NX unit accepts data only on 4K page boundaries */ #define NX842_HW_PAGE_SHIFT SHIFT_4K #define NX842_HW_PAGE_SIZE (ASM_CONST(1) << NX842_HW_PAGE_SHIFT) #define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1)) enum nx842_status { UNAVAILABLE, AVAILABLE }; struct ibm_nx842_counters { atomic64_t comp_complete; atomic64_t comp_failed; atomic64_t decomp_complete; atomic64_t decomp_failed; atomic64_t swdecomp; atomic64_t comp_times[32]; atomic64_t decomp_times[32]; }; static struct nx842_devdata { struct vio_dev *vdev; struct device *dev; struct ibm_nx842_counters *counters; unsigned int max_sg_len; unsigned int max_sync_size; unsigned int max_sync_sg; enum nx842_status status; } __rcu *devdata; static DEFINE_SPINLOCK(devdata_mutex); #define NX842_COUNTER_INC(_x) \ static inline void nx842_inc_##_x( \ const struct nx842_devdata *dev) { \ if (dev) \ atomic64_inc(&dev->counters->_x); \ } NX842_COUNTER_INC(comp_complete); NX842_COUNTER_INC(comp_failed); NX842_COUNTER_INC(decomp_complete); NX842_COUNTER_INC(decomp_failed); NX842_COUNTER_INC(swdecomp); #define NX842_HIST_SLOTS 16 static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time) { int bucket = fls(time); if (bucket) bucket = min((NX842_HIST_SLOTS - 1), bucket - 1); atomic64_inc(×[bucket]); } /* NX unit operation flags */ #define NX842_OP_COMPRESS 0x0 #define NX842_OP_CRC 0x1 #define NX842_OP_DECOMPRESS 0x2 #define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC) #define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC) #define NX842_OP_ASYNC (1<<23) #define NX842_OP_NOTIFY (1<<22) #define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8) static unsigned long nx842_get_desired_dma(struct vio_dev *viodev) { /* No use of DMA mappings within the driver. */ return 0; } struct nx842_slentry { unsigned long ptr; /* Absolute address (use virt_to_abs()) */ unsigned long len; }; /* pHyp scatterlist entry */ struct nx842_scatterlist { int entry_nr; /* number of slentries */ struct nx842_slentry *entries; /* ptr to array of slentries */ }; /* Does not include sizeof(entry_nr) in the size */ static inline unsigned long nx842_get_scatterlist_size( struct nx842_scatterlist *sl) { return sl->entry_nr * sizeof(struct nx842_slentry); } static int nx842_build_scatterlist(unsigned long buf, int len, struct nx842_scatterlist *sl) { unsigned long nextpage; struct nx842_slentry *entry; sl->entry_nr = 0; entry = sl->entries; while (len) { entry->ptr = virt_to_abs(buf); nextpage = ALIGN(buf + 1, NX842_HW_PAGE_SIZE); if (nextpage < buf + len) { /* we aren't at the end yet */ if (IS_ALIGNED(buf, NX842_HW_PAGE_SIZE)) /* we are in the middle (or beginning) */ entry->len = NX842_HW_PAGE_SIZE; else /* we are at the beginning */ entry->len = nextpage - buf; } else { /* at the end */ entry->len = len; } len -= entry->len; buf += entry->len; sl->entry_nr++; entry++; } return 0; } /* * Working memory for software decompression */ struct sw842_fifo { union { char f8[256][8]; char f4[512][4]; }; char f2[256][2]; unsigned char f84_full; unsigned char f2_full; unsigned char f8_count; unsigned char f2_count; unsigned int f4_count; }; /* * Working memory for crypto API */ struct nx842_workmem { char bounce[PAGE_SIZE]; /* bounce buffer for decompression input */ union { /* hardware working memory */ struct { /* scatterlist */ char slin[SIZE_4K]; char slout[SIZE_4K]; /* coprocessor status/parameter block */ struct nx_csbcpb csbcpb; }; /* software working memory */ struct sw842_fifo swfifo; /* software decompression fifo */ }; }; int nx842_get_workmem_size(void) { return sizeof(struct nx842_workmem) + NX842_HW_PAGE_SIZE; } EXPORT_SYMBOL_GPL(nx842_get_workmem_size); int nx842_get_workmem_size_aligned(void) { return sizeof(struct nx842_workmem); } EXPORT_SYMBOL_GPL(nx842_get_workmem_size_aligned); static int nx842_validate_result(struct device *dev, struct cop_status_block *csb) { /* The csb must be valid after returning from vio_h_cop_sync */ if (!NX842_CSBCBP_VALID_CHK(csb->valid)) { dev_err(dev, "%s: cspcbp not valid upon completion.\n", __func__); dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n", csb->valid, csb->crb_seq_number, csb->completion_code, csb->completion_extension); dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n", csb->processed_byte_count, (unsigned long)csb->address); return -EIO; } /* Check return values from the hardware in the CSB */ switch (csb->completion_code) { case 0: /* Completed without error */ break; case 64: /* Target bytes > Source bytes during compression */ case 13: /* Output buffer too small */ dev_dbg(dev, "%s: Compression output larger than input\n", __func__); return -ENOSPC; case 66: /* Input data contains an illegal template field */ case 67: /* Template indicates data past the end of the input stream */ dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n", __func__, csb->completion_code); return -EINVAL; default: dev_dbg(dev, "%s: Unspecified error (code:%d)\n", __func__, csb->completion_code); return -EIO; } /* Hardware sanity check */ if (!NX842_CSBCPB_CE2(csb->completion_extension)) { dev_err(dev, "%s: No error returned by hardware, but " "data returned is unusable, contact support.\n" "(Additional info: csbcbp->processed bytes " "does not specify processed bytes for the " "target buffer.)\n", __func__); return -EIO; } return 0; } /** * nx842_compress - Compress data using the 842 algorithm * * Compression provide by the NX842 coprocessor on IBM Power systems. * The input buffer is compressed and the result is stored in the * provided output buffer. * * Upon return from this function @outlen contains the length of the * compressed data. If there is an error then @outlen will be 0 and an * error will be specified by the return code from this function. * * @in: Pointer to input buffer, must be page aligned * @inlen: Length of input buffer, must be PAGE_SIZE * @out: Pointer to output buffer * @outlen: Length of output buffer * @wrkmem: ptr to buffer for working memory, size determined by * nx842_get_workmem_size() * * Returns: * 0 Success, output of length @outlen stored in the buffer at @out * -ENOMEM Unable to allocate internal buffers * -ENOSPC Output buffer is to small * -EMSGSIZE XXX Difficult to describe this limitation * -EIO Internal error * -ENODEV Hardware unavailable */ int nx842_compress(const unsigned char *in, unsigned int inlen, unsigned char *out, unsigned int *outlen, void *wmem) { struct nx842_header *hdr; struct nx842_devdata *local_devdata; struct device *dev = NULL; struct nx842_workmem *workmem; struct nx842_scatterlist slin, slout; struct nx_csbcpb *csbcpb; int ret = 0, max_sync_size, i, bytesleft, size, hdrsize; unsigned long inbuf, outbuf, padding; struct vio_pfo_op op = { .done = NULL, .handle = 0, .timeout = 0, }; unsigned long start_time = get_tb(); /* * Make sure input buffer is 64k page aligned. This is assumed since * this driver is designed for page compression only (for now). This * is very nice since we can now use direct DDE(s) for the input and * the alignment is guaranteed. */ inbuf = (unsigned long)in; if (!IS_ALIGNED(inbuf, PAGE_SIZE) || inlen != PAGE_SIZE) return -EINVAL; rcu_read_lock(); local_devdata = rcu_dereference(devdata); if (!local_devdata || !local_devdata->dev) { rcu_read_unlock(); return -ENODEV; } max_sync_size = local_devdata->max_sync_size; dev = local_devdata->dev; /* Create the header */ hdr = (struct nx842_header *)out; hdr->blocks_nr = PAGE_SIZE / max_sync_size; hdrsize = nx842_header_size(hdr); outbuf = (unsigned long)out + hdrsize; bytesleft = *outlen - hdrsize; /* Init scatterlist */ workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem, NX842_HW_PAGE_SIZE); slin.entries = (struct nx842_slentry *)workmem->slin; slout.entries = (struct nx842_slentry *)workmem->slout; /* Init operation */ op.flags = NX842_OP_COMPRESS; csbcpb = &workmem->csbcpb; memset(csbcpb, 0, sizeof(*csbcpb)); op.csbcpb = virt_to_abs(csbcpb); op.out = virt_to_abs(slout.entries); for (i = 0; i < hdr->blocks_nr; i++) { /* * Aligning the output blocks to 128 bytes does waste space, * but it prevents the need for bounce buffers and memory * copies. It also simplifies the code a lot. In the worst * case (64k page, 4k max_sync_size), you lose up to * (128*16)/64k = ~3% the compression factor. For 64k * max_sync_size, the loss would be at most 128/64k = ~0.2%. */ padding = ALIGN(outbuf, IO_BUFFER_ALIGN) - outbuf; outbuf += padding; bytesleft -= padding; if (i == 0) /* save offset into first block in header */ hdr->offset = padding + hdrsize; if (bytesleft <= 0) { ret = -ENOSPC; goto unlock; } /* * NOTE: If the default max_sync_size is changed from 4k * to 64k, remove the "likely" case below, since a * scatterlist will always be needed. */ if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) { /* Create direct DDE */ op.in = virt_to_abs(inbuf); op.inlen = max_sync_size; } else { /* Create indirect DDE (scatterlist) */ nx842_build_scatterlist(inbuf, max_sync_size, &slin); op.in = virt_to_abs(slin.entries); op.inlen = -nx842_get_scatterlist_size(&slin); } /* * If max_sync_size != NX842_HW_PAGE_SIZE, an indirect * DDE is required for the outbuf. * If max_sync_size == NX842_HW_PAGE_SIZE, outbuf must * also be page aligned (1 in 128/4k=32 chance) in order * to use a direct DDE. * This is unlikely, just use an indirect DDE always. */ nx842_build_scatterlist(outbuf, min(bytesleft, max_sync_size), &slout); /* op.out set before loop */ op.outlen = -nx842_get_scatterlist_size(&slout); /* Send request to pHyp */ ret = vio_h_cop_sync(local_devdata->vdev, &op); /* Check for pHyp error */ if (ret) { dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n", __func__, ret, op.hcall_err); ret = -EIO; goto unlock; } /* Check for hardware error */ ret = nx842_validate_result(dev, &csbcpb->csb); if (ret && ret != -ENOSPC) goto unlock; /* Handle incompressible data */ if (unlikely(ret == -ENOSPC)) { if (bytesleft < max_sync_size) { /* * Not enough space left in the output buffer * to store uncompressed block */ goto unlock; } else { /* Store incompressible block */ memcpy((void *)outbuf, (void *)inbuf, max_sync_size); hdr->sizes[i] = -max_sync_size; outbuf += max_sync_size; bytesleft -= max_sync_size; /* Reset ret, incompressible data handled */ ret = 0; } } else { /* Normal case, compression was successful */ size = csbcpb->csb.processed_byte_count; dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, size); hdr->sizes[i] = size; outbuf += size; bytesleft -= size; } inbuf += max_sync_size; } *outlen = (unsigned int)(outbuf - (unsigned long)out); unlock: if (ret) nx842_inc_comp_failed(local_devdata); else { nx842_inc_comp_complete(local_devdata); ibm_nx842_incr_hist(local_devdata->counters->comp_times, (get_tb() - start_time) / tb_ticks_per_usec); } rcu_read_unlock(); return ret; } EXPORT_SYMBOL_GPL(nx842_compress); static int sw842_decompress(const unsigned char *, int, unsigned char *, int *, const void *); /** * nx842_decompress - Decompress data using the 842 algorithm * * Decompression provide by the NX842 coprocessor on IBM Power systems. * The input buffer is decompressed and the result is stored in the * provided output buffer. The size allocated to the output buffer is * provided by the caller of this function in @outlen. Upon return from * this function @outlen contains the length of the decompressed data. * If there is an error then @outlen will be 0 and an error will be * specified by the return code from this function. * * @in: Pointer to input buffer, will use bounce buffer if not 128 byte * aligned * @inlen: Length of input buffer * @out: Pointer to output buffer, must be page aligned * @outlen: Length of output buffer, must be PAGE_SIZE * @wrkmem: ptr to buffer for working memory, size determined by * nx842_get_workmem_size() * * Returns: * 0 Success, output of length @outlen stored in the buffer at @out * -ENODEV Hardware decompression device is unavailable * -ENOMEM Unable to allocate internal buffers * -ENOSPC Output buffer is to small * -EINVAL Bad input data encountered when attempting decompress * -EIO Internal error */ int nx842_decompress(const unsigned char *in, unsigned int inlen, unsigned char *out, unsigned int *outlen, void *wmem) { struct nx842_header *hdr; struct nx842_devdata *local_devdata; struct device *dev = NULL; struct nx842_workmem *workmem; struct nx842_scatterlist slin, slout; struct nx_csbcpb *csbcpb; int ret = 0, i, size, max_sync_size; unsigned long inbuf, outbuf; struct vio_pfo_op op = { .done = NULL, .handle = 0, .timeout = 0, }; unsigned long start_time = get_tb(); /* Ensure page alignment and size */ outbuf = (unsigned long)out; if (!IS_ALIGNED(outbuf, PAGE_SIZE) || *outlen != PAGE_SIZE) return -EINVAL; rcu_read_lock(); local_devdata = rcu_dereference(devdata); if (local_devdata) dev = local_devdata->dev; /* Get header */ hdr = (struct nx842_header *)in; workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem, NX842_HW_PAGE_SIZE); inbuf = (unsigned long)in + hdr->offset; if (likely(!IS_ALIGNED(inbuf, IO_BUFFER_ALIGN))) { /* Copy block(s) into bounce buffer for alignment */ memcpy(workmem->bounce, in + hdr->offset, inlen - hdr->offset); inbuf = (unsigned long)workmem->bounce; } /* Init scatterlist */ slin.entries = (struct nx842_slentry *)workmem->slin; slout.entries = (struct nx842_slentry *)workmem->slout; /* Init operation */ op.flags = NX842_OP_DECOMPRESS; csbcpb = &workmem->csbcpb; memset(csbcpb, 0, sizeof(*csbcpb)); op.csbcpb = virt_to_abs(csbcpb); /* * max_sync_size may have changed since compression, * so we can't read it from the device info. We need * to derive it from hdr->blocks_nr. */ max_sync_size = PAGE_SIZE / hdr->blocks_nr; for (i = 0; i < hdr->blocks_nr; i++) { /* Skip padding */ inbuf = ALIGN(inbuf, IO_BUFFER_ALIGN); if (hdr->sizes[i] < 0) { /* Negative sizes indicate uncompressed data blocks */ size = abs(hdr->sizes[i]); memcpy((void *)outbuf, (void *)inbuf, size); outbuf += size; inbuf += size; continue; } if (!dev) goto sw; /* * The better the compression, the more likely the "likely" * case becomes. */ if (likely((inbuf & NX842_HW_PAGE_MASK) == ((inbuf + hdr->sizes[i] - 1) & NX842_HW_PAGE_MASK))) { /* Create direct DDE */ op.in = virt_to_abs(inbuf); op.inlen = hdr->sizes[i]; } else { /* Create indirect DDE (scatterlist) */ nx842_build_scatterlist(inbuf, hdr->sizes[i] , &slin); op.in = virt_to_abs(slin.entries); op.inlen = -nx842_get_scatterlist_size(&slin); } /* * NOTE: If the default max_sync_size is changed from 4k * to 64k, remove the "likely" case below, since a * scatterlist will always be needed. */ if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) { /* Create direct DDE */ op.out = virt_to_abs(outbuf); op.outlen = max_sync_size; } else { /* Create indirect DDE (scatterlist) */ nx842_build_scatterlist(outbuf, max_sync_size, &slout); op.out = virt_to_abs(slout.entries); op.outlen = -nx842_get_scatterlist_size(&slout); } /* Send request to pHyp */ ret = vio_h_cop_sync(local_devdata->vdev, &op); /* Check for pHyp error */ if (ret) { dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n", __func__, ret, op.hcall_err); dev = NULL; goto sw; } /* Check for hardware error */ ret = nx842_validate_result(dev, &csbcpb->csb); if (ret) { dev = NULL; goto sw; } /* HW decompression success */ inbuf += hdr->sizes[i]; outbuf += csbcpb->csb.processed_byte_count; continue; sw: /* software decompression */ size = max_sync_size; ret = sw842_decompress( (unsigned char *)inbuf, hdr->sizes[i], (unsigned char *)outbuf, &size, wmem); if (ret) pr_debug("%s: sw842_decompress failed with %d\n", __func__, ret); if (ret) { if (ret != -ENOSPC && ret != -EINVAL && ret != -EMSGSIZE) ret = -EIO; goto unlock; } /* SW decompression success */ inbuf += hdr->sizes[i]; outbuf += size; } *outlen = (unsigned int)(outbuf - (unsigned long)out); unlock: if (ret) /* decompress fail */ nx842_inc_decomp_failed(local_devdata); else { if (!dev) /* software decompress */ nx842_inc_swdecomp(local_devdata); nx842_inc_decomp_complete(local_devdata); ibm_nx842_incr_hist(local_devdata->counters->decomp_times, (get_tb() - start_time) / tb_ticks_per_usec); } rcu_read_unlock(); return ret; } EXPORT_SYMBOL_GPL(nx842_decompress); /** * nx842_OF_set_defaults -- Set default (disabled) values for devdata * * @devdata - struct nx842_devdata to update * * Returns: * 0 on success * -ENOENT if @devdata ptr is NULL */ static int nx842_OF_set_defaults(struct nx842_devdata *devdata) { if (devdata) { devdata->max_sync_size = 0; devdata->max_sync_sg = 0; devdata->max_sg_len = 0; devdata->status = UNAVAILABLE; return 0; } else return -ENOENT; } /** * nx842_OF_upd_status -- Update the device info from OF status prop * * The status property indicates if the accelerator is enabled. If the * device is in the OF tree it indicates that the hardware is present. * The status field indicates if the device is enabled when the status * is 'okay'. Otherwise the device driver will be disabled. * * @devdata - struct nx842_devdata to update * @prop - struct property point containing the maxsyncop for the update * * Returns: * 0 - Device is available * -EINVAL - Device is not available */ static int nx842_OF_upd_status(struct nx842_devdata *devdata, struct property *prop) { int ret = 0; const char *status = (const char *)prop->value; if (!strncmp(status, "okay", (size_t)prop->length)) { devdata->status = AVAILABLE; } else { dev_info(devdata->dev, "%s: status '%s' is not 'okay'\n", __func__, status); devdata->status = UNAVAILABLE; } return ret; } /** * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop * * Definition of the 'ibm,max-sg-len' OF property: * This field indicates the maximum byte length of a scatter list * for the platform facility. It is a single cell encoded as with encode-int. * * Example: * # od -x ibm,max-sg-len * 0000000 0000 0ff0 * * In this example, the maximum byte length of a scatter list is * 0x0ff0 (4,080). * * @devdata - struct nx842_devdata to update * @prop - struct property point containing the maxsyncop for the update * * Returns: * 0 on success * -EINVAL on failure */ static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata, struct property *prop) { int ret = 0; const int *maxsglen = prop->value; if (prop->length != sizeof(*maxsglen)) { dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__); dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__, prop->length, sizeof(*maxsglen)); ret = -EINVAL; } else { devdata->max_sg_len = (unsigned int)min(*maxsglen, (int)NX842_HW_PAGE_SIZE); } return ret; } /** * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop * * Definition of the 'ibm,max-sync-cop' OF property: * Two series of cells. The first series of cells represents the maximums * that can be synchronously compressed. The second series of cells * represents the maximums that can be synchronously decompressed. * 1. The first cell in each series contains the count of the number of * data length, scatter list elements pairs that follow – each being * of the form * a. One cell data byte length * b. One cell total number of scatter list elements * * Example: * # od -x ibm,max-sync-cop * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001 * 0000020 0000 1000 0000 01fe * * In this example, compression supports 0x1000 (4,096) data byte length * and 0x1fe (510) total scatter list elements. Decompression supports * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list * elements. * * @devdata - struct nx842_devdata to update * @prop - struct property point containing the maxsyncop for the update * * Returns: * 0 on success * -EINVAL on failure */ static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata, struct property *prop) { int ret = 0; const struct maxsynccop_t { int comp_elements; int comp_data_limit; int comp_sg_limit; int decomp_elements; int decomp_data_limit; int decomp_sg_limit; } *maxsynccop; if (prop->length != sizeof(*maxsynccop)) { dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__); dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length, sizeof(*maxsynccop)); ret = -EINVAL; goto out; } maxsynccop = (const struct maxsynccop_t *)prop->value; /* Use one limit rather than separate limits for compression and * decompression. Set a maximum for this so as not to exceed the * size that the header can support and round the value down to * the hardware page size (4K) */ devdata->max_sync_size = (unsigned int)min(maxsynccop->comp_data_limit, maxsynccop->decomp_data_limit); devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size, SIZE_64K); if (devdata->max_sync_size < SIZE_4K) { dev_err(devdata->dev, "%s: hardware max data size (%u) is " "less than the driver minimum, unable to use " "the hardware device\n", __func__, devdata->max_sync_size); ret = -EINVAL; goto out; } devdata->max_sync_sg = (unsigned int)min(maxsynccop->comp_sg_limit, maxsynccop->decomp_sg_limit); if (devdata->max_sync_sg < 1) { dev_err(devdata->dev, "%s: hardware max sg size (%u) is " "less than the driver minimum, unable to use " "the hardware device\n", __func__, devdata->max_sync_sg); ret = -EINVAL; goto out; } out: return ret; } /** * * nx842_OF_upd -- Handle OF properties updates for the device. * * Set all properties from the OF tree. Optionally, a new property * can be provided by the @new_prop pointer to overwrite an existing value. * The device will remain disabled until all values are valid, this function * will return an error for updates unless all values are valid. * * @new_prop: If not NULL, this property is being updated. If NULL, update * all properties from the current values in the OF tree. * * Returns: * 0 - Success * -ENOMEM - Could not allocate memory for new devdata structure * -EINVAL - property value not found, new_prop is not a recognized * property for the device or property value is not valid. * -ENODEV - Device is not available */ static int nx842_OF_upd(struct property *new_prop) { struct nx842_devdata *old_devdata = NULL; struct nx842_devdata *new_devdata = NULL; struct device_node *of_node = NULL; struct property *status = NULL; struct property *maxsglen = NULL; struct property *maxsyncop = NULL; int ret = 0; unsigned long flags; spin_lock_irqsave(&devdata_mutex, flags); old_devdata = rcu_dereference_check(devdata, lockdep_is_held(&devdata_mutex)); if (old_devdata) of_node = old_devdata->dev->of_node; if (!old_devdata || !of_node) { pr_err("%s: device is not available\n", __func__); spin_unlock_irqrestore(&devdata_mutex, flags); return -ENODEV; } new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS); if (!new_devdata) { dev_err(old_devdata->dev, "%s: Could not allocate memory for device data\n", __func__); ret = -ENOMEM; goto error_out; } memcpy(new_devdata, old_devdata, sizeof(*old_devdata)); new_devdata->counters = old_devdata->counters; /* Set ptrs for existing properties */ status = of_find_property(of_node, "status", NULL); maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL); maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL); if (!status || !maxsglen || !maxsyncop) { dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__); ret = -EINVAL; goto error_out; } /* Set ptr to new property if provided */ if (new_prop) { /* Single property */ if (!strncmp(new_prop->name, "status", new_prop->length)) { status = new_prop; } else if (!strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length)) { maxsglen = new_prop; } else if (!strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)) { maxsyncop = new_prop; } else { /* * Skip the update, the property being updated * has no impact. */ goto out; } } /* Perform property updates */ ret = nx842_OF_upd_status(new_devdata, status); if (ret) goto error_out; ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen); if (ret) goto error_out; ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop); if (ret) goto error_out; out: dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n", __func__, new_devdata->max_sync_size, old_devdata->max_sync_size); dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n", __func__, new_devdata->max_sync_sg, old_devdata->max_sync_sg); dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n", __func__, new_devdata->max_sg_len, old_devdata->max_sg_len); rcu_assign_pointer(devdata, new_devdata); spin_unlock_irqrestore(&devdata_mutex, flags); synchronize_rcu(); dev_set_drvdata(new_devdata->dev, new_devdata); kfree(old_devdata); return 0; error_out: if (new_devdata) { dev_info(old_devdata->dev, "%s: device disabled\n", __func__); nx842_OF_set_defaults(new_devdata); rcu_assign_pointer(devdata, new_devdata); spin_unlock_irqrestore(&devdata_mutex, flags); synchronize_rcu(); dev_set_drvdata(new_devdata->dev, new_devdata); kfree(old_devdata); } else { dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__); spin_unlock_irqrestore(&devdata_mutex, flags); } if (!ret) ret = -EINVAL; return ret; } /** * nx842_OF_notifier - Process updates to OF properties for the device * * @np: notifier block * @action: notifier action * @update: struct pSeries_reconfig_prop_update pointer if action is * PSERIES_UPDATE_PROPERTY * * Returns: * NOTIFY_OK on success * NOTIFY_BAD encoded with error number on failure, use * notifier_to_errno() to decode this value */ static int nx842_OF_notifier(struct notifier_block *np, unsigned long action, void *update) { struct pSeries_reconfig_prop_update *upd; struct nx842_devdata *local_devdata; struct device_node *node = NULL; upd = (struct pSeries_reconfig_prop_update *)update; rcu_read_lock(); local_devdata = rcu_dereference(devdata); if (local_devdata) node = local_devdata->dev->of_node; if (local_devdata && action == PSERIES_UPDATE_PROPERTY && !strcmp(upd->node->name, node->name)) { rcu_read_unlock(); nx842_OF_upd(upd->property); } else rcu_read_unlock(); return NOTIFY_OK; } static struct notifier_block nx842_of_nb = { .notifier_call = nx842_OF_notifier, }; #define nx842_counter_read(_name) \ static ssize_t nx842_##_name##_show(struct device *dev, \ struct device_attribute *attr, \ char *buf) { \ struct nx842_devdata *local_devdata; \ int p = 0; \ rcu_read_lock(); \ local_devdata = rcu_dereference(devdata); \ if (local_devdata) \ p = snprintf(buf, PAGE_SIZE, "%ld\n", \ atomic64_read(&local_devdata->counters->_name)); \ rcu_read_unlock(); \ return p; \ } #define NX842DEV_COUNTER_ATTR_RO(_name) \ nx842_counter_read(_name); \ static struct device_attribute dev_attr_##_name = __ATTR(_name, \ 0444, \ nx842_##_name##_show,\ NULL); NX842DEV_COUNTER_ATTR_RO(comp_complete); NX842DEV_COUNTER_ATTR_RO(comp_failed); NX842DEV_COUNTER_ATTR_RO(decomp_complete); NX842DEV_COUNTER_ATTR_RO(decomp_failed); NX842DEV_COUNTER_ATTR_RO(swdecomp); static ssize_t nx842_timehist_show(struct device *, struct device_attribute *, char *); static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444, nx842_timehist_show, NULL); static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times, 0444, nx842_timehist_show, NULL); static ssize_t nx842_timehist_show(struct device *dev, struct device_attribute *attr, char *buf) { char *p = buf; struct nx842_devdata *local_devdata; atomic64_t *times; int bytes_remain = PAGE_SIZE; int bytes; int i; rcu_read_lock(); local_devdata = rcu_dereference(devdata); if (!local_devdata) { rcu_read_unlock(); return 0; } if (attr == &dev_attr_comp_times) times = local_devdata->counters->comp_times; else if (attr == &dev_attr_decomp_times) times = local_devdata->counters->decomp_times; else { rcu_read_unlock(); return 0; } for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) { bytes = snprintf(p, bytes_remain, "%u-%uus:\t%ld\n", i ? (2<<(i-1)) : 0, (2<vdev != NULL) { dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__); ret = -1; goto error_unlock; } dev_set_drvdata(&viodev->dev, NULL); new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS); if (!new_devdata) { dev_err(&viodev->dev, "%s: Could not allocate memory for device data\n", __func__); ret = -ENOMEM; goto error_unlock; } new_devdata->counters = kzalloc(sizeof(*new_devdata->counters), GFP_NOFS); if (!new_devdata->counters) { dev_err(&viodev->dev, "%s: Could not allocate memory for performance counters\n", __func__); ret = -ENOMEM; goto error_unlock; } new_devdata->vdev = viodev; new_devdata->dev = &viodev->dev; nx842_OF_set_defaults(new_devdata); rcu_assign_pointer(devdata, new_devdata); spin_unlock_irqrestore(&devdata_mutex, flags); synchronize_rcu(); kfree(old_devdata); pSeries_reconfig_notifier_register(&nx842_of_nb); ret = nx842_OF_upd(NULL); if (ret && ret != -ENODEV) { dev_err(&viodev->dev, "could not parse device tree. %d\n", ret); ret = -1; goto error; } rcu_read_lock(); if (dev_set_drvdata(&viodev->dev, rcu_dereference(devdata))) { rcu_read_unlock(); dev_err(&viodev->dev, "failed to set driver data for device\n"); ret = -1; goto error; } rcu_read_unlock(); if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) { dev_err(&viodev->dev, "could not create sysfs device attributes\n"); ret = -1; goto error; } return 0; error_unlock: spin_unlock_irqrestore(&devdata_mutex, flags); if (new_devdata) kfree(new_devdata->counters); kfree(new_devdata); error: return ret; } static int __exit nx842_remove(struct vio_dev *viodev) { struct nx842_devdata *old_devdata; unsigned long flags; pr_info("Removing IBM Power 842 compression device\n"); sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group); spin_lock_irqsave(&devdata_mutex, flags); old_devdata = rcu_dereference_check(devdata, lockdep_is_held(&devdata_mutex)); pSeries_reconfig_notifier_unregister(&nx842_of_nb); rcu_assign_pointer(devdata, NULL); spin_unlock_irqrestore(&devdata_mutex, flags); synchronize_rcu(); dev_set_drvdata(&viodev->dev, NULL); if (old_devdata) kfree(old_devdata->counters); kfree(old_devdata); return 0; } static struct vio_device_id nx842_driver_ids[] = { {"ibm,compression-v1", "ibm,compression"}, {"", ""}, }; static struct vio_driver nx842_driver = { .name = MODULE_NAME, .probe = nx842_probe, .remove = nx842_remove, .get_desired_dma = nx842_get_desired_dma, .id_table = nx842_driver_ids, }; static int __init nx842_init(void) { struct nx842_devdata *new_devdata; pr_info("Registering IBM Power 842 compression driver\n"); RCU_INIT_POINTER(devdata, NULL); new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL); if (!new_devdata) { pr_err("Could not allocate memory for device data\n"); return -ENOMEM; } new_devdata->status = UNAVAILABLE; RCU_INIT_POINTER(devdata, new_devdata); return vio_register_driver(&nx842_driver); } module_init(nx842_init); static void __exit nx842_exit(void) { struct nx842_devdata *old_devdata; unsigned long flags; pr_info("Exiting IBM Power 842 compression driver\n"); spin_lock_irqsave(&devdata_mutex, flags); old_devdata = rcu_dereference_check(devdata, lockdep_is_held(&devdata_mutex)); rcu_assign_pointer(devdata, NULL); spin_unlock_irqrestore(&devdata_mutex, flags); synchronize_rcu(); if (old_devdata) dev_set_drvdata(old_devdata->dev, NULL); kfree(old_devdata); vio_unregister_driver(&nx842_driver); } module_exit(nx842_exit); /********************************* * 842 software decompressor *********************************/ typedef int (*sw842_template_op)(const char **, int *, unsigned char **, struct sw842_fifo *); static int sw842_data8(const char **, int *, unsigned char **, struct sw842_fifo *); static int sw842_data4(const char **, int *, unsigned char **, struct sw842_fifo *); static int sw842_data2(const char **, int *, unsigned char **, struct sw842_fifo *); static int sw842_ptr8(const char **, int *, unsigned char **, struct sw842_fifo *); static int sw842_ptr4(const char **, int *, unsigned char **, struct sw842_fifo *); static int sw842_ptr2(const char **, int *, unsigned char **, struct sw842_fifo *); /* special templates */ #define SW842_TMPL_REPEAT 0x1B #define SW842_TMPL_ZEROS 0x1C #define SW842_TMPL_EOF 0x1E static sw842_template_op sw842_tmpl_ops[26][4] = { { sw842_data8, NULL}, /* 0 (00000) */ { sw842_data4, sw842_data2, sw842_ptr2, NULL}, { sw842_data4, sw842_ptr2, sw842_data2, NULL}, { sw842_data4, sw842_ptr2, sw842_ptr2, NULL}, { sw842_data4, sw842_ptr4, NULL}, { sw842_data2, sw842_ptr2, sw842_data4, NULL}, { sw842_data2, sw842_ptr2, sw842_data2, sw842_ptr2}, { sw842_data2, sw842_ptr2, sw842_ptr2, sw842_data2}, { sw842_data2, sw842_ptr2, sw842_ptr2, sw842_ptr2,}, { sw842_data2, sw842_ptr2, sw842_ptr4, NULL}, { sw842_ptr2, sw842_data2, sw842_data4, NULL}, /* 10 (01010) */ { sw842_ptr2, sw842_data4, sw842_ptr2, NULL}, { sw842_ptr2, sw842_data2, sw842_ptr2, sw842_data2}, { sw842_ptr2, sw842_data2, sw842_ptr2, sw842_ptr2}, { sw842_ptr2, sw842_data2, sw842_ptr4, NULL}, { sw842_ptr2, sw842_ptr2, sw842_data4, NULL}, { sw842_ptr2, sw842_ptr2, sw842_data2, sw842_ptr2}, { sw842_ptr2, sw842_ptr2, sw842_ptr2, sw842_data2}, { sw842_ptr2, sw842_ptr2, sw842_ptr2, sw842_ptr2}, { sw842_ptr2, sw842_ptr2, sw842_ptr4, NULL}, { sw842_ptr4, sw842_data4, NULL}, /* 20 (10100) */ { sw842_ptr4, sw842_data2, sw842_ptr2, NULL}, { sw842_ptr4, sw842_ptr2, sw842_data2, NULL}, { sw842_ptr4, sw842_ptr2, sw842_ptr2, NULL}, { sw842_ptr4, sw842_ptr4, NULL}, { sw842_ptr8, NULL} }; /* Software decompress helpers */ static uint8_t sw842_get_byte(const char *buf, int bit) { uint8_t tmpl; uint16_t tmp; tmp = htons(*(uint16_t *)(buf)); tmp = (uint16_t)(tmp << bit); tmp = ntohs(tmp); memcpy(&tmpl, &tmp, 1); return tmpl; } static uint8_t sw842_get_template(const char **buf, int *bit) { uint8_t byte; byte = sw842_get_byte(*buf, *bit); byte = byte >> 3; byte &= 0x1F; *buf += (*bit + 5) / 8; *bit = (*bit + 5) % 8; return byte; } /* repeat_count happens to be 5-bit too (like the template) */ static uint8_t sw842_get_repeat_count(const char **buf, int *bit) { uint8_t byte; byte = sw842_get_byte(*buf, *bit); byte = byte >> 2; byte &= 0x3F; *buf += (*bit + 6) / 8; *bit = (*bit + 6) % 8; return byte; } static uint8_t sw842_get_ptr2(const char **buf, int *bit) { uint8_t ptr; ptr = sw842_get_byte(*buf, *bit); (*buf)++; return ptr; } static uint16_t sw842_get_ptr4(const char **buf, int *bit, struct sw842_fifo *fifo) { uint16_t ptr; ptr = htons(*(uint16_t *)(*buf)); ptr = (uint16_t)(ptr << *bit); ptr = ptr >> 7; ptr &= 0x01FF; *buf += (*bit + 9) / 8; *bit = (*bit + 9) % 8; return ptr; } static uint8_t sw842_get_ptr8(const char **buf, int *bit, struct sw842_fifo *fifo) { return sw842_get_ptr2(buf, bit); } /* Software decompress template ops */ static int sw842_data8(const char **inbuf, int *inbit, unsigned char **outbuf, struct sw842_fifo *fifo) { int ret; ret = sw842_data4(inbuf, inbit, outbuf, fifo); if (ret) return ret; ret = sw842_data4(inbuf, inbit, outbuf, fifo); return ret; } static int sw842_data4(const char **inbuf, int *inbit, unsigned char **outbuf, struct sw842_fifo *fifo) { int ret; ret = sw842_data2(inbuf, inbit, outbuf, fifo); if (ret) return ret; ret = sw842_data2(inbuf, inbit, outbuf, fifo); return ret; } static int sw842_data2(const char **inbuf, int *inbit, unsigned char **outbuf, struct sw842_fifo *fifo) { **outbuf = sw842_get_byte(*inbuf, *inbit); (*inbuf)++; (*outbuf)++; **outbuf = sw842_get_byte(*inbuf, *inbit); (*inbuf)++; (*outbuf)++; return 0; } static int sw842_ptr8(const char **inbuf, int *inbit, unsigned char **outbuf, struct sw842_fifo *fifo) { uint8_t ptr; ptr = sw842_get_ptr8(inbuf, inbit, fifo); if (!fifo->f84_full && (ptr >= fifo->f8_count)) return 1; memcpy(*outbuf, fifo->f8[ptr], 8); *outbuf += 8; return 0; } static int sw842_ptr4(const char **inbuf, int *inbit, unsigned char **outbuf, struct sw842_fifo *fifo) { uint16_t ptr; ptr = sw842_get_ptr4(inbuf, inbit, fifo); if (!fifo->f84_full && (ptr >= fifo->f4_count)) return 1; memcpy(*outbuf, fifo->f4[ptr], 4); *outbuf += 4; return 0; } static int sw842_ptr2(const char **inbuf, int *inbit, unsigned char **outbuf, struct sw842_fifo *fifo) { uint8_t ptr; ptr = sw842_get_ptr2(inbuf, inbit); if (!fifo->f2_full && (ptr >= fifo->f2_count)) return 1; memcpy(*outbuf, fifo->f2[ptr], 2); *outbuf += 2; return 0; } static void sw842_copy_to_fifo(const char *buf, struct sw842_fifo *fifo) { unsigned char initial_f2count = fifo->f2_count; memcpy(fifo->f8[fifo->f8_count], buf, 8); fifo->f4_count += 2; fifo->f8_count += 1; if (!fifo->f84_full && fifo->f4_count >= 512) { fifo->f84_full = 1; fifo->f4_count /= 512; } memcpy(fifo->f2[fifo->f2_count++], buf, 2); memcpy(fifo->f2[fifo->f2_count++], buf + 2, 2); memcpy(fifo->f2[fifo->f2_count++], buf + 4, 2); memcpy(fifo->f2[fifo->f2_count++], buf + 6, 2); if (fifo->f2_count < initial_f2count) fifo->f2_full = 1; } static int sw842_decompress(const unsigned char *src, int srclen, unsigned char *dst, int *destlen, const void *wrkmem) { uint8_t tmpl; const char *inbuf; int inbit = 0; unsigned char *outbuf, *outbuf_end, *origbuf, *prevbuf; const char *inbuf_end; sw842_template_op op; int opindex; int i, repeat_count; struct sw842_fifo *fifo; int ret = 0; fifo = &((struct nx842_workmem *)(wrkmem))->swfifo; memset(fifo, 0, sizeof(*fifo)); origbuf = NULL; inbuf = src; inbuf_end = src + srclen; outbuf = dst; outbuf_end = dst + *destlen; while ((tmpl = sw842_get_template(&inbuf, &inbit)) != SW842_TMPL_EOF) { if (inbuf >= inbuf_end) { ret = -EINVAL; goto out; } opindex = 0; prevbuf = origbuf; origbuf = outbuf; switch (tmpl) { case SW842_TMPL_REPEAT: if (prevbuf == NULL) { ret = -EINVAL; goto out; } repeat_count = sw842_get_repeat_count(&inbuf, &inbit) + 1; /* Did the repeat count advance past the end of input */ if (inbuf > inbuf_end) { ret = -EINVAL; goto out; } for (i = 0; i < repeat_count; i++) { /* Would this overflow the output buffer */ if ((outbuf + 8) > outbuf_end) { ret = -ENOSPC; goto out; } memcpy(outbuf, prevbuf, 8); sw842_copy_to_fifo(outbuf, fifo); outbuf += 8; } break; case SW842_TMPL_ZEROS: /* Would this overflow the output buffer */ if ((outbuf + 8) > outbuf_end) { ret = -ENOSPC; goto out; } memset(outbuf, 0, 8); sw842_copy_to_fifo(outbuf, fifo); outbuf += 8; break; default: if (tmpl > 25) { ret = -EINVAL; goto out; } /* Does this go past the end of the input buffer */ if ((inbuf + 2) > inbuf_end) { ret = -EINVAL; goto out; } /* Would this overflow the output buffer */ if ((outbuf + 8) > outbuf_end) { ret = -ENOSPC; goto out; } while (opindex < 4 && (op = sw842_tmpl_ops[tmpl][opindex++]) != NULL) { ret = (*op)(&inbuf, &inbit, &outbuf, fifo); if (ret) { ret = -EINVAL; goto out; } sw842_copy_to_fifo(origbuf, fifo); } } } out: if (!ret) *destlen = (unsigned int)(outbuf - dst); else *destlen = 0; return ret; }