package/ubsec_ssb/src/openbsd/ubsec.c

   1 /*      $OpenBSD: ubsec.c,v 1.140 2007/10/01 15:34:48 krw Exp $ */
   2
   3 /*
   4  * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
   5  * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
   6  * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  19  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  20  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
  21  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  22  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  23  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  25  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27  * POSSIBILITY OF SUCH DAMAGE.
  28  *
  29  * Effort sponsored in part by the Defense Advanced Research Projects
  30  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  31  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  32  *
  33  */
  34
  35 #undef UBSEC_DEBUG
  36
  37 /*
  38  * uBsec 5[56]01, 58xx hardware crypto accelerator
  39  */
  40
  41 #include <sys/param.h>
  42 #include <sys/systm.h>
  43 #include <sys/proc.h>
  44 #include <sys/errno.h>
  45 #include <sys/malloc.h>
  46 #include <sys/kernel.h>
  47 #include <sys/mbuf.h>
  48 #include <sys/device.h>
  49 #include <sys/queue.h>
  50
  51 #include <crypto/cryptodev.h>
  52 #include <crypto/cryptosoft.h>
  53 #include <dev/rndvar.h>
  54 #include <crypto/md5.h>
  55 #include <crypto/sha1.h>
  56
  57 #include <dev/pci/pcireg.h>
  58 #include <dev/pci/pcivar.h>
  59 #include <dev/pci/pcidevs.h>
  60
  61 #include <dev/pci/ubsecreg.h>
  62 #include <dev/pci/ubsecvar.h>
  63
  64 /*
  65  * Prototypes and count for the pci_device structure
  66  */
  67 int ubsec_probe(struct device *, void *, void *);
  68 void ubsec_attach(struct device *, struct device *, void *);
  69 void ubsec_reset_board(struct ubsec_softc *);
  70 void ubsec_init_board(struct ubsec_softc *);
  71 void ubsec_init_pciregs(struct pci_attach_args *pa);
  72 void ubsec_cleanchip(struct ubsec_softc *);
  73 void ubsec_totalreset(struct ubsec_softc *);
  74 int  ubsec_free_q(struct ubsec_softc*, struct ubsec_q *);
  75
  76 struct cfattach ubsec_ca = {
  77         sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach,
  78 };
  79
  80 struct cfdriver ubsec_cd = {
  81         0, "ubsec", DV_DULL
  82 };
  83
  84 int     ubsec_intr(void *);
  85 int     ubsec_newsession(u_int32_t *, struct cryptoini *);
  86 int     ubsec_freesession(u_int64_t);
  87 int     ubsec_process(struct cryptop *);
  88 void    ubsec_callback(struct ubsec_softc *, struct ubsec_q *);
  89 void    ubsec_feed(struct ubsec_softc *);
  90 void    ubsec_mcopy(struct mbuf *, struct mbuf *, int, int);
  91 void    ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *);
  92 void    ubsec_feed2(struct ubsec_softc *);
  93 void    ubsec_rng(void *);
  94 int     ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
  95     struct ubsec_dma_alloc *, int);
  96 void    ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *);
  97 int     ubsec_dmamap_aligned(bus_dmamap_t);
  98
  99 int     ubsec_kprocess(struct cryptkop *);
 100 struct ubsec_softc *ubsec_kfind(struct cryptkop *);
 101 int     ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *);
 102 int     ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *);
 103 int     ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *);
 104 void    ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *);
 105 int     ubsec_ksigbits(struct crparam *);
 106 void    ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 107 void    ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 108
 109 /* DEBUG crap... */
 110 void ubsec_dump_pb(struct ubsec_pktbuf *);
 111 void ubsec_dump_mcr(struct ubsec_mcr *);
 112 void ubsec_dump_ctx2(struct ubsec_ctx_keyop *);
 113
 114 #define READ_REG(sc,r) \
 115         bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
 116
 117 #define WRITE_REG(sc,reg,val) \
 118         bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
 119
 120 #define SWAP32(x) (x) = htole32(ntohl((x)))
 121 #define HTOLE32(x) (x) = htole32(x)
 122
 123
 124 struct ubsec_stats ubsecstats;
 125
 126 const struct pci_matchid ubsec_devices[] = {
 127         { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5501 },
 128         { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5601 },
 129         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5801 },
 130         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5802 },
 131         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5805 },
 132         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5820 },
 133         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5821 },
 134         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5822 },
 135         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5823 },
 136         { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_SCA1K },
 137         { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_5821 },
 138 };
 139
 140 int
 141 ubsec_probe(struct device *parent, void *match, void *aux)
 142 {
 143         return (pci_matchbyid((struct pci_attach_args *)aux, ubsec_devices,
 144             sizeof(ubsec_devices)/sizeof(ubsec_devices[0])));
 145 }
 146
 147 void
 148 ubsec_attach(struct device *parent, struct device *self, void *aux)
 149 {
 150         struct ubsec_softc *sc = (struct ubsec_softc *)self;
 151         struct pci_attach_args *pa = aux;
 152         pci_chipset_tag_t pc = pa->pa_pc;
 153         pci_intr_handle_t ih;
 154         const char *intrstr = NULL;
 155         struct ubsec_dma *dmap;
 156         bus_size_t iosize;
 157         u_int32_t i;
 158         int algs[CRYPTO_ALGORITHM_MAX + 1];
 159         int kalgs[CRK_ALGORITHM_MAX + 1];
 160
 161         SIMPLEQ_INIT(&sc->sc_queue);
 162         SIMPLEQ_INIT(&sc->sc_qchip);
 163         SIMPLEQ_INIT(&sc->sc_queue2);
 164         SIMPLEQ_INIT(&sc->sc_qchip2);
 165         SIMPLEQ_INIT(&sc->sc_q2free);
 166
 167         sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR;
 168
 169         if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_BLUESTEEL &&
 170             PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_BLUESTEEL_5601)
 171                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 172
 173         if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_BROADCOM &&
 174             (PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_BROADCOM_5802 ||
 175              PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_BROADCOM_5805))
 176                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 177
 178         if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_BROADCOM &&
 179             (PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_BROADCOM_5820 ||
 180              PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_BROADCOM_5822 ||
 181              PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_BROADCOM_5823))
 182                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 183                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 184
 185         if ((PCI_VENDOR(pa->pa_id) == PCI_VENDOR_BROADCOM &&
 186              PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_BROADCOM_5821) ||
 187             (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_SUN &&
 188              (PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_SUN_SCA1K ||
 189               PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_SUN_5821))) {
 190                 sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY |
 191                     BS_STAT_MCR2_ALLEMPTY;
 192                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 193                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 194         }
 195
 196         if (pci_mapreg_map(pa, BS_BAR, PCI_MAPREG_TYPE_MEM, 0,
 197             &sc->sc_st, &sc->sc_sh, NULL, &iosize, 0)) {
 198                 printf(": can't find mem space\n");
 199                 return;
 200         }
 201         sc->sc_dmat = pa->pa_dmat;
 202
 203         if (pci_intr_map(pa, &ih)) {
 204                 printf(": couldn't map interrupt\n");
 205                 bus_space_unmap(sc->sc_st, sc->sc_sh, iosize);
 206                 return;
 207         }
 208         intrstr = pci_intr_string(pc, ih);
 209         sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, ubsec_intr, sc,
 210             self->dv_xname);
 211         if (sc->sc_ih == NULL) {
 212                 printf(": couldn't establish interrupt");
 213                 if (intrstr != NULL)
 214                         printf(" at %s", intrstr);
 215                 printf("\n");
 216                 bus_space_unmap(sc->sc_st, sc->sc_sh, iosize);
 217                 return;
 218         }
 219
 220         sc->sc_cid = crypto_get_driverid(0);
 221         if (sc->sc_cid < 0) {
 222                 pci_intr_disestablish(pc, sc->sc_ih);
 223                 bus_space_unmap(sc->sc_st, sc->sc_sh, iosize);
 224                 return;
 225         }
 226
 227         SIMPLEQ_INIT(&sc->sc_freequeue);
 228         dmap = sc->sc_dmaa;
 229         for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
 230                 struct ubsec_q *q;
 231
 232                 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q),
 233                     M_DEVBUF, M_NOWAIT);
 234                 if (q == NULL) {
 235                         printf(": can't allocate queue buffers\n");
 236                         break;
 237                 }
 238
 239                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
 240                     &dmap->d_alloc, 0)) {
 241                         printf(": can't allocate dma buffers\n");
 242                         free(q, M_DEVBUF);
 243                         break;
 244                 }
 245                 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
 246
 247                 q->q_dma = dmap;
 248                 sc->sc_queuea[i] = q;
 249
 250                 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 251         }
 252
 253         bzero(algs, sizeof(algs));
 254         algs[CRYPTO_3DES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
 255         algs[CRYPTO_DES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
 256         algs[CRYPTO_MD5_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
 257         algs[CRYPTO_SHA1_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
 258         crypto_register(sc->sc_cid, algs, ubsec_newsession,
 259             ubsec_freesession, ubsec_process);
 260
 261         /*
 262          * Reset Broadcom chip
 263          */
 264         ubsec_reset_board(sc);
 265
 266         /*
 267          * Init Broadcom specific PCI settings
 268          */
 269         ubsec_init_pciregs(pa);
 270
 271         /*
 272          * Init Broadcom chip
 273          */
 274         ubsec_init_board(sc);
 275
 276         printf(": 3DES MD5 SHA1");
 277
 278 #ifndef UBSEC_NO_RNG
 279         if (sc->sc_flags & UBS_FLAGS_RNG) {
 280                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
 281
 282                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 283                     &sc->sc_rng.rng_q.q_mcr, 0))
 284                         goto skip_rng;
 285
 286                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
 287                     &sc->sc_rng.rng_q.q_ctx, 0)) {
 288                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 289                         goto skip_rng;
 290                 }
 291
 292                 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
 293                     UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) {
 294                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 295                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 296                         goto skip_rng;
 297                 }
 298
 299                 timeout_set(&sc->sc_rngto, ubsec_rng, sc);
 300                 if (hz >= 100)
 301                         sc->sc_rnghz = hz / 100;
 302                 else
 303                         sc->sc_rnghz = 1;
 304                 timeout_add(&sc->sc_rngto, sc->sc_rnghz);
 305                 printf(" RNG");
 306 skip_rng:
 307         ;
 308         }
 309 #endif /* UBSEC_NO_RNG */
 310
 311         if (sc->sc_flags & UBS_FLAGS_KEY) {
 312                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
 313
 314                 bzero(kalgs, sizeof(kalgs));
 315                 kalgs[CRK_MOD_EXP] = CRYPTO_ALG_FLAG_SUPPORTED;
 316 #if 0
 317                 kalgs[CRK_MOD_EXP_CRT] = CRYPTO_ALG_FLAG_SUPPORTED;
 318 #endif
 319
 320                 crypto_kregister(sc->sc_cid, kalgs, ubsec_kprocess);
 321                 printf(" PK");
 322         }
 323
 324         printf(", %s\n", intrstr);
 325 }
 326
 327 /*
 328  * UBSEC Interrupt routine
 329  */
 330 int
 331 ubsec_intr(void *arg)
 332 {
 333         struct ubsec_softc *sc = arg;
 334         volatile u_int32_t stat;
 335         struct ubsec_q *q;
 336         struct ubsec_dma *dmap;
 337         int npkts = 0, i;
 338
 339         stat = READ_REG(sc, BS_STAT);
 340
 341         stat &= sc->sc_statmask;
 342         if (stat == 0)
 343                 return (0);
 344
 345         WRITE_REG(sc, BS_STAT, stat);           /* IACK */
 346
 347         /*
 348          * Check to see if we have any packets waiting for us
 349          */
 350         if ((stat & BS_STAT_MCR1_DONE)) {
 351                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
 352                         q = SIMPLEQ_FIRST(&sc->sc_qchip);
 353                         dmap = q->q_dma;
 354
 355                         if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
 356                                 break;
 357
 358                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
 359
 360                         npkts = q->q_nstacked_mcrs;
 361                         /*
 362                          * search for further sc_qchip ubsec_q's that share
 363                          * the same MCR, and complete them too, they must be
 364                          * at the top.
 365                          */
 366                         for (i = 0; i < npkts; i++) {
 367                                 if(q->q_stacked_mcr[i])
 368                                         ubsec_callback(sc, q->q_stacked_mcr[i]);
 369                                 else
 370                                         break;
 371                         }
 372                         ubsec_callback(sc, q);
 373                 }
 374
 375                 /*
 376                  * Don't send any more packet to chip if there has been
 377                  * a DMAERR.
 378                  */
 379                 if (!(stat & BS_STAT_DMAERR))
 380                         ubsec_feed(sc);
 381         }
 382
 383         /*
 384          * Check to see if we have any key setups/rng's waiting for us
 385          */
 386         if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) &&
 387             (stat & BS_STAT_MCR2_DONE)) {
 388                 struct ubsec_q2 *q2;
 389                 struct ubsec_mcr *mcr;
 390
 391                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
 392                         q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
 393
 394                         bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map,
 395                             0, q2->q_mcr.dma_map->dm_mapsize,
 396                             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 397
 398                         mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
 399                         if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) {
 400                                 bus_dmamap_sync(sc->sc_dmat,
 401                                     q2->q_mcr.dma_map, 0,
 402                                     q2->q_mcr.dma_map->dm_mapsize,
 403                                     BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 404                                 break;
 405                         }
 406                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q_next);
 407                         ubsec_callback2(sc, q2);
 408                         /*
 409                          * Don't send any more packet to chip if there has been
 410                          * a DMAERR.
 411                          */
 412                         if (!(stat & BS_STAT_DMAERR))
 413                                 ubsec_feed2(sc);
 414                 }
 415         }
 416
 417         /*
 418          * Check to see if we got any DMA Error
 419          */
 420         if (stat & BS_STAT_DMAERR) {
 421 #ifdef UBSEC_DEBUG
 422                 volatile u_int32_t a = READ_REG(sc, BS_ERR);
 423
 424                 printf("%s: dmaerr %s@%08x\n", sc->sc_dv.dv_xname,
 425                     (a & BS_ERR_READ) ? "read" : "write", a & BS_ERR_ADDR);
 426 #endif /* UBSEC_DEBUG */
 427                 ubsecstats.hst_dmaerr++;
 428                 ubsec_totalreset(sc);
 429                 ubsec_feed(sc);
 430         }
 431
 432         return (1);
 433 }
 434
 435 /*
 436  * ubsec_feed() - aggregate and post requests to chip
 437  *                It is assumed that the caller set splnet()
 438  */
 439 void
 440 ubsec_feed(struct ubsec_softc *sc)
 441 {
 442 #ifdef UBSEC_DEBUG
 443         static int max;
 444 #endif /* UBSEC_DEBUG */
 445         struct ubsec_q *q, *q2;
 446         int npkts, i;
 447         void *v;
 448         u_int32_t stat;
 449
 450         npkts = sc->sc_nqueue;
 451         if (npkts > UBS_MAX_AGGR)
 452                 npkts = UBS_MAX_AGGR;
 453         if (npkts < 2)
 454                 goto feed1;
 455
 456         if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
 457                 if(stat & BS_STAT_DMAERR) {
 458                         ubsec_totalreset(sc);
 459                         ubsecstats.hst_dmaerr++;
 460                 }
 461                 return;
 462         }
 463
 464 #ifdef UBSEC_DEBUG
 465         printf("merging %d records\n", npkts);
 466
 467         /* XXX temporary aggregation statistics reporting code */
 468         if (max < npkts) {
 469                 max = npkts;
 470                 printf("%s: new max aggregate %d\n", sc->sc_dv.dv_xname, max);
 471         }
 472 #endif /* UBSEC_DEBUG */
 473
 474         q = SIMPLEQ_FIRST(&sc->sc_queue);
 475         SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
 476         --sc->sc_nqueue;
 477
 478         bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
 479             0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
 480         if (q->q_dst_map != NULL)
 481                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
 482                     0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
 483
 484         q->q_nstacked_mcrs = npkts - 1;         /* Number of packets stacked */
 485
 486         for (i = 0; i < q->q_nstacked_mcrs; i++) {
 487                 q2 = SIMPLEQ_FIRST(&sc->sc_queue);
 488                 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
 489                     0, q2->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
 490                 if (q2->q_dst_map != NULL)
 491                         bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
 492                             0, q2->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
 493                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
 494                 --sc->sc_nqueue;
 495
 496                 v = ((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) -
 497                     sizeof(struct ubsec_mcr_add);
 498                 bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add));
 499                 q->q_stacked_mcr[i] = q2;
 500         }
 501         q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
 502         SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 503         bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map,
 504             0, q->q_dma->d_alloc.dma_map->dm_mapsize,
 505             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 506         WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 507             offsetof(struct ubsec_dmachunk, d_mcr));
 508         return;
 509
 510 feed1:
 511         while (!SIMPLEQ_EMPTY(&sc->sc_queue)) {
 512                 if ((stat = READ_REG(sc, BS_STAT)) &
 513                     (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
 514                         if(stat & BS_STAT_DMAERR) {
 515                                 ubsec_totalreset(sc);
 516                                 ubsecstats.hst_dmaerr++;
 517                         }
 518                         break;
 519                 }
 520
 521                 q = SIMPLEQ_FIRST(&sc->sc_queue);
 522
 523                 bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
 524                     0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
 525                 if (q->q_dst_map != NULL)
 526                         bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
 527                             0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
 528                 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map,
 529                     0, q->q_dma->d_alloc.dma_map->dm_mapsize,
 530                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 531
 532                 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 533                     offsetof(struct ubsec_dmachunk, d_mcr));
 534 #ifdef UBSEC_DEBUG
 535                 printf("feed: q->chip %p %08x\n", q,
 536                     (u_int32_t)q->q_dma->d_alloc.dma_paddr);
 537 #endif /* UBSEC_DEBUG */
 538                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
 539                 --sc->sc_nqueue;
 540                 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 541         }
 542 }
 543
 544 /*
 545  * Allocate a new 'session' and return an encoded session id.  'sidp'
 546  * contains our registration id, and should contain an encoded session
 547  * id on successful allocation.
 548  */
 549 int
 550 ubsec_newsession(u_int32_t *sidp, struct cryptoini *cri)
 551 {
 552         struct cryptoini *c, *encini = NULL, *macini = NULL;
 553         struct ubsec_softc *sc = NULL;
 554         struct ubsec_session *ses = NULL;
 555         MD5_CTX md5ctx;
 556         SHA1_CTX sha1ctx;
 557         int i, sesn;
 558
 559         if (sidp == NULL || cri == NULL)
 560                 return (EINVAL);
 561
 562         for (i = 0; i < ubsec_cd.cd_ndevs; i++) {
 563                 sc = ubsec_cd.cd_devs[i];
 564                 if (sc == NULL || sc->sc_cid == (*sidp))
 565                         break;
 566         }
 567         if (sc == NULL)
 568                 return (EINVAL);
 569
 570         for (c = cri; c != NULL; c = c->cri_next) {
 571                 if (c->cri_alg == CRYPTO_MD5_HMAC ||
 572                     c->cri_alg == CRYPTO_SHA1_HMAC) {
 573                         if (macini)
 574                                 return (EINVAL);
 575                         macini = c;
 576                 } else if (c->cri_alg == CRYPTO_DES_CBC ||
 577                     c->cri_alg == CRYPTO_3DES_CBC) {
 578                         if (encini)
 579                                 return (EINVAL);
 580                         encini = c;
 581                 } else
 582                         return (EINVAL);
 583         }
 584         if (encini == NULL && macini == NULL)
 585                 return (EINVAL);
 586
 587         if (sc->sc_sessions == NULL) {
 588                 ses = sc->sc_sessions = (struct ubsec_session *)malloc(
 589                     sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 590                 if (ses == NULL)
 591                         return (ENOMEM);
 592                 sesn = 0;
 593                 sc->sc_nsessions = 1;
 594         } else {
 595                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 596                         if (sc->sc_sessions[sesn].ses_used == 0) {
 597                                 ses = &sc->sc_sessions[sesn];
 598                                 break;
 599                         }
 600                 }
 601
 602                 if (ses == NULL) {
 603                         sesn = sc->sc_nsessions;
 604                         ses = (struct ubsec_session *)malloc((sesn + 1) *
 605                             sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 606                         if (ses == NULL)
 607                                 return (ENOMEM);
 608                         bcopy(sc->sc_sessions, ses, sesn *
 609                             sizeof(struct ubsec_session));
 610                         bzero(sc->sc_sessions, sesn *
 611                             sizeof(struct ubsec_session));
 612                         free(sc->sc_sessions, M_DEVBUF);
 613                         sc->sc_sessions = ses;
 614                         ses = &sc->sc_sessions[sesn];
 615                         sc->sc_nsessions++;
 616                 }
 617         }
 618
 619         bzero(ses, sizeof(struct ubsec_session));
 620         ses->ses_used = 1;
 621         if (encini) {
 622                 /* get an IV, network byte order */
 623                 arc4random_bytes(ses->ses_iv, sizeof(ses->ses_iv));
 624
 625                 /* Go ahead and compute key in ubsec's byte order */
 626                 if (encini->cri_alg == CRYPTO_DES_CBC) {
 627                         bcopy(encini->cri_key, &ses->ses_deskey[0], 8);
 628                         bcopy(encini->cri_key, &ses->ses_deskey[2], 8);
 629                         bcopy(encini->cri_key, &ses->ses_deskey[4], 8);
 630                 } else
 631                         bcopy(encini->cri_key, ses->ses_deskey, 24);
 632
 633                 SWAP32(ses->ses_deskey[0]);
 634                 SWAP32(ses->ses_deskey[1]);
 635                 SWAP32(ses->ses_deskey[2]);
 636                 SWAP32(ses->ses_deskey[3]);
 637                 SWAP32(ses->ses_deskey[4]);
 638                 SWAP32(ses->ses_deskey[5]);
 639         }
 640
 641         if (macini) {
 642                 for (i = 0; i < macini->cri_klen / 8; i++)
 643                         macini->cri_key[i] ^= HMAC_IPAD_VAL;
 644
 645                 if (macini->cri_alg == CRYPTO_MD5_HMAC) {
 646                         MD5Init(&md5ctx);
 647                         MD5Update(&md5ctx, macini->cri_key,
 648                             macini->cri_klen / 8);
 649                         MD5Update(&md5ctx, hmac_ipad_buffer,
 650                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 651                         bcopy(md5ctx.state, ses->ses_hminner,
 652                             sizeof(md5ctx.state));
 653                 } else {
 654                         SHA1Init(&sha1ctx);
 655                         SHA1Update(&sha1ctx, macini->cri_key,
 656                             macini->cri_klen / 8);
 657                         SHA1Update(&sha1ctx, hmac_ipad_buffer,
 658                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 659                         bcopy(sha1ctx.state, ses->ses_hminner,
 660                             sizeof(sha1ctx.state));
 661                 }
 662
 663                 for (i = 0; i < macini->cri_klen / 8; i++)
 664                         macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
 665
 666                 if (macini->cri_alg == CRYPTO_MD5_HMAC) {
 667                         MD5Init(&md5ctx);
 668                         MD5Update(&md5ctx, macini->cri_key,
 669                             macini->cri_klen / 8);
 670                         MD5Update(&md5ctx, hmac_opad_buffer,
 671                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 672                         bcopy(md5ctx.state, ses->ses_hmouter,
 673                             sizeof(md5ctx.state));
 674                 } else {
 675                         SHA1Init(&sha1ctx);
 676                         SHA1Update(&sha1ctx, macini->cri_key,
 677                             macini->cri_klen / 8);
 678                         SHA1Update(&sha1ctx, hmac_opad_buffer,
 679                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 680                         bcopy(sha1ctx.state, ses->ses_hmouter,
 681                             sizeof(sha1ctx.state));
 682                 }
 683
 684                 for (i = 0; i < macini->cri_klen / 8; i++)
 685                         macini->cri_key[i] ^= HMAC_OPAD_VAL;
 686         }
 687
 688         *sidp = UBSEC_SID(sc->sc_dv.dv_unit, sesn);
 689         return (0);
 690 }
 691
 692 /*
 693  * Deallocate a session.
 694  */
 695 int
 696 ubsec_freesession(u_int64_t tid)
 697 {
 698         struct ubsec_softc *sc;
 699         int card, session;
 700         u_int32_t sid = ((u_int32_t)tid) & 0xffffffff;
 701
 702         card = UBSEC_CARD(sid);
 703         if (card >= ubsec_cd.cd_ndevs || ubsec_cd.cd_devs[card] == NULL)
 704                 return (EINVAL);
 705         sc = ubsec_cd.cd_devs[card];
 706         session = UBSEC_SESSION(sid);
 707         bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
 708         return (0);
 709 }
 710
 711 int
 712 ubsec_process(struct cryptop *crp)
 713 {
 714         struct ubsec_q *q = NULL;
 715         int card, err = 0, i, j, s, nicealign;
 716         struct ubsec_softc *sc;
 717         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
 718         int encoffset = 0, macoffset = 0, cpskip, cpoffset;
 719         int sskip, dskip, stheend, dtheend;
 720         int16_t coffset;
 721         struct ubsec_session *ses;
 722         struct ubsec_pktctx ctx;
 723         struct ubsec_dma *dmap = NULL;
 724
 725         if (crp == NULL || crp->crp_callback == NULL) {
 726                 ubsecstats.hst_invalid++;
 727                 return (EINVAL);
 728         }
 729         card = UBSEC_CARD(crp->crp_sid);
 730         if (card >= ubsec_cd.cd_ndevs || ubsec_cd.cd_devs[card] == NULL) {
 731                 ubsecstats.hst_invalid++;
 732                 return (EINVAL);
 733         }
 734
 735         sc = ubsec_cd.cd_devs[card];
 736
 737         s = splnet();
 738
 739         if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
 740                 ubsecstats.hst_queuefull++;
 741                 splx(s);
 742                 err = ENOMEM;
 743                 goto errout2;
 744         }
 745
 746         q = SIMPLEQ_FIRST(&sc->sc_freequeue);
 747         SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
 748         splx(s);
 749
 750         dmap = q->q_dma; /* Save dma pointer */
 751         bzero(q, sizeof(struct ubsec_q));
 752         bzero(&ctx, sizeof(ctx));
 753
 754         q->q_sesn = UBSEC_SESSION(crp->crp_sid);
 755         q->q_dma = dmap;
 756         ses = &sc->sc_sessions[q->q_sesn];
 757
 758         if (crp->crp_flags & CRYPTO_F_IMBUF) {
 759                 q->q_src_m = (struct mbuf *)crp->crp_buf;
 760                 q->q_dst_m = (struct mbuf *)crp->crp_buf;
 761         } else if (crp->crp_flags & CRYPTO_F_IOV) {
 762                 q->q_src_io = (struct uio *)crp->crp_buf;
 763                 q->q_dst_io = (struct uio *)crp->crp_buf;
 764         } else {
 765                 err = EINVAL;
 766                 goto errout;    /* XXX we don't handle contiguous blocks! */
 767         }
 768
 769         bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
 770
 771         dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
 772         dmap->d_dma->d_mcr.mcr_flags = 0;
 773         q->q_crp = crp;
 774
 775         crd1 = crp->crp_desc;
 776         if (crd1 == NULL) {
 777                 err = EINVAL;
 778                 goto errout;
 779         }
 780         crd2 = crd1->crd_next;
 781
 782         if (crd2 == NULL) {
 783                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
 784                     crd1->crd_alg == CRYPTO_SHA1_HMAC) {
 785                         maccrd = crd1;
 786                         enccrd = NULL;
 787                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
 788                     crd1->crd_alg == CRYPTO_3DES_CBC) {
 789                         maccrd = NULL;
 790                         enccrd = crd1;
 791                 } else {
 792                         err = EINVAL;
 793                         goto errout;
 794                 }
 795         } else {
 796                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
 797                     crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
 798                     (crd2->crd_alg == CRYPTO_DES_CBC ||
 799                         crd2->crd_alg == CRYPTO_3DES_CBC) &&
 800                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
 801                         maccrd = crd1;
 802                         enccrd = crd2;
 803                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
 804                     crd1->crd_alg == CRYPTO_3DES_CBC) &&
 805                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
 806                         crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
 807                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
 808                         enccrd = crd1;
 809                         maccrd = crd2;
 810                 } else {
 811                         /*
 812                          * We cannot order the ubsec as requested
 813                          */
 814                         err = EINVAL;
 815                         goto errout;
 816                 }
 817         }
 818
 819         if (enccrd) {
 820                 encoffset = enccrd->crd_skip;
 821                 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);
 822
 823                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
 824                         q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
 825
 826                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
 827                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
 828                         else {
 829                                 ctx.pc_iv[0] = ses->ses_iv[0];
 830                                 ctx.pc_iv[1] = ses->ses_iv[1];
 831                         }
 832
 833                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
 834                                 if (crp->crp_flags & CRYPTO_F_IMBUF)
 835                                         m_copyback(q->q_src_m,
 836                                             enccrd->crd_inject,
 837                                             8, ctx.pc_iv);
 838                                 else if (crp->crp_flags & CRYPTO_F_IOV)
 839                                         cuio_copyback(q->q_src_io,
 840                                             enccrd->crd_inject,
 841                                             8, ctx.pc_iv);
 842                         }
 843                 } else {
 844                         ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND);
 845
 846                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
 847                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
 848                         else if (crp->crp_flags & CRYPTO_F_IMBUF)
 849                                 m_copydata(q->q_src_m, enccrd->crd_inject,
 850                                     8, (caddr_t)ctx.pc_iv);
 851                         else if (crp->crp_flags & CRYPTO_F_IOV)
 852                                 cuio_copydata(q->q_src_io,
 853                                     enccrd->crd_inject, 8,
 854                                     (caddr_t)ctx.pc_iv);
 855                 }
 856
 857                 ctx.pc_deskey[0] = ses->ses_deskey[0];
 858                 ctx.pc_deskey[1] = ses->ses_deskey[1];
 859                 ctx.pc_deskey[2] = ses->ses_deskey[2];
 860                 ctx.pc_deskey[3] = ses->ses_deskey[3];
 861                 ctx.pc_deskey[4] = ses->ses_deskey[4];
 862                 ctx.pc_deskey[5] = ses->ses_deskey[5];
 863                 SWAP32(ctx.pc_iv[0]);
 864                 SWAP32(ctx.pc_iv[1]);
 865         }
 866
 867         if (maccrd) {
 868                 macoffset = maccrd->crd_skip;
 869
 870                 if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
 871                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5);
 872                 else
 873                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
 874
 875                 for (i = 0; i < 5; i++) {
 876                         ctx.pc_hminner[i] = ses->ses_hminner[i];
 877                         ctx.pc_hmouter[i] = ses->ses_hmouter[i];
 878
 879                         HTOLE32(ctx.pc_hminner[i]);
 880                         HTOLE32(ctx.pc_hmouter[i]);
 881                 }
 882         }
 883
 884         if (enccrd && maccrd) {
 885                 /*
 886                  * ubsec cannot handle packets where the end of encryption
 887                  * and authentication are not the same, or where the
 888                  * encrypted part begins before the authenticated part.
 889                  */
 890                 if (((encoffset + enccrd->crd_len) !=
 891                     (macoffset + maccrd->crd_len)) ||
 892                     (enccrd->crd_skip < maccrd->crd_skip)) {
 893                         err = EINVAL;
 894                         goto errout;
 895                 }
 896                 sskip = maccrd->crd_skip;
 897                 cpskip = dskip = enccrd->crd_skip;
 898                 stheend = maccrd->crd_len;
 899                 dtheend = enccrd->crd_len;
 900                 coffset = enccrd->crd_skip - maccrd->crd_skip;
 901                 cpoffset = cpskip + dtheend;
 902 #ifdef UBSEC_DEBUG
 903                 printf("mac: skip %d, len %d, inject %d\n",
 904                     maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
 905                 printf("enc: skip %d, len %d, inject %d\n",
 906                     enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
 907                 printf("src: skip %d, len %d\n", sskip, stheend);
 908                 printf("dst: skip %d, len %d\n", dskip, dtheend);
 909                 printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
 910                     coffset, stheend, cpskip, cpoffset);
 911 #endif
 912         } else {
 913                 cpskip = dskip = sskip = macoffset + encoffset;
 914                 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
 915                 cpoffset = cpskip + dtheend;
 916                 coffset = 0;
 917         }
 918         ctx.pc_offset = htole16(coffset >> 2);
 919
 920         if (bus_dmamap_create(sc->sc_dmat, 0xfff0, UBS_MAX_SCATTER,
 921                 0xfff0, 0, BUS_DMA_NOWAIT, &q->q_src_map) != 0) {
 922                 err = ENOMEM;
 923                 goto errout;
 924         }
 925         if (crp->crp_flags & CRYPTO_F_IMBUF) {
 926                 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
 927                     q->q_src_m, BUS_DMA_NOWAIT) != 0) {
 928                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
 929                         q->q_src_map = NULL;
 930                         err = ENOMEM;
 931                         goto errout;
 932                 }
 933         } else if (crp->crp_flags & CRYPTO_F_IOV) {
 934                 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
 935                     q->q_src_io, BUS_DMA_NOWAIT) != 0) {
 936                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
 937                         q->q_src_map = NULL;
 938                         err = ENOMEM;
 939                         goto errout;
 940                 }
 941         }
 942         nicealign = ubsec_dmamap_aligned(q->q_src_map);
 943
 944         dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
 945
 946 #ifdef UBSEC_DEBUG
 947         printf("src skip: %d\n", sskip);
 948 #endif
 949         for (i = j = 0; i < q->q_src_map->dm_nsegs; i++) {
 950                 struct ubsec_pktbuf *pb;
 951                 bus_size_t packl = q->q_src_map->dm_segs[i].ds_len;
 952                 bus_addr_t packp = q->q_src_map->dm_segs[i].ds_addr;
 953
 954                 if (sskip >= packl) {
 955                         sskip -= packl;
 956                         continue;
 957                 }
 958
 959                 packl -= sskip;
 960                 packp += sskip;
 961                 sskip = 0;
 962
 963                 if (packl > 0xfffc) {
 964                         err = EIO;
 965                         goto errout;
 966                 }
 967
 968                 if (j == 0)
 969                         pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
 970                 else
 971                         pb = &dmap->d_dma->d_sbuf[j - 1];
 972
 973                 pb->pb_addr = htole32(packp);
 974
 975                 if (stheend) {
 976                         if (packl > stheend) {
 977                                 pb->pb_len = htole32(stheend);
 978                                 stheend = 0;
 979                         } else {
 980                                 pb->pb_len = htole32(packl);
 981                                 stheend -= packl;
 982                         }
 983                 } else
 984                         pb->pb_len = htole32(packl);
 985
 986                 if ((i + 1) == q->q_src_map->dm_nsegs)
 987                         pb->pb_next = 0;
 988                 else
 989                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
 990                             offsetof(struct ubsec_dmachunk, d_sbuf[j]));
 991                 j++;
 992         }
 993
 994         if (enccrd == NULL && maccrd != NULL) {
 995                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
 996                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
 997                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next =
 998                     htole32(dmap->d_alloc.dma_paddr +
 999                     offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1000 #ifdef UBSEC_DEBUG
1001                 printf("opkt: %x %x %x\n",
1002                     dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
1003                     dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
1004                     dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
1005 #endif
1006         } else {
1007                 if (crp->crp_flags & CRYPTO_F_IOV) {
1008                         if (!nicealign) {
1009                                 err = EINVAL;
1010                                 goto errout;
1011                         }
1012                         if (bus_dmamap_create(sc->sc_dmat, 0xfff0,
1013                             UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT,
1014                             &q->q_dst_map) != 0) {
1015                                 err = ENOMEM;
1016                                 goto errout;
1017                         }
1018                         if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
1019                             q->q_dst_io, BUS_DMA_NOWAIT) != 0) {
1020                                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1021                                 q->q_dst_map = NULL;
1022                                 goto errout;
1023                         }
1024                 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1025                         if (nicealign) {
1026                                 q->q_dst_m = q->q_src_m;
1027                                 q->q_dst_map = q->q_src_map;
1028                         } else {
1029                                 int totlen, len;
1030                                 struct mbuf *m, *top, **mp;
1031
1032                                 totlen = q->q_src_map->dm_mapsize;
1033                                 if (q->q_src_m->m_flags & M_PKTHDR) {
1034                                         len = MHLEN;
1035                                         MGETHDR(m, M_DONTWAIT, MT_DATA);
1036                                 } else {
1037                                         len = MLEN;
1038                                         MGET(m, M_DONTWAIT, MT_DATA);
1039                                 }
1040                                 if (m == NULL) {
1041                                         err = ENOMEM;
1042                                         goto errout;
1043                                 }
1044                                 if (len == MHLEN)
1045                                         M_DUP_PKTHDR(m, q->q_src_m);
1046                                 if (totlen >= MINCLSIZE) {
1047                                         MCLGET(m, M_DONTWAIT);
1048                                         if (m->m_flags & M_EXT)
1049                                                 len = MCLBYTES;
1050                                 }
1051                                 m->m_len = len;
1052                                 top = NULL;
1053                                 mp = &top;
1054
1055                                 while (totlen > 0) {
1056                                         if (top) {
1057                                                 MGET(m, M_DONTWAIT, MT_DATA);
1058                                                 if (m == NULL) {
1059                                                         m_freem(top);
1060                                                         err = ENOMEM;
1061                                                         goto errout;
1062                                                 }
1063                                                 len = MLEN;
1064                                         }
1065                                         if (top && totlen >= MINCLSIZE) {
1066                                                 MCLGET(m, M_DONTWAIT);
1067                                                 if (m->m_flags & M_EXT)
1068                                                         len = MCLBYTES;
1069                                         }
1070                                         m->m_len = len = min(totlen, len);
1071                                         totlen -= len;
1072                                         *mp = m;
1073                                         mp = &m->m_next;
1074                                 }
1075                                 q->q_dst_m = top;
1076                                 ubsec_mcopy(q->q_src_m, q->q_dst_m,
1077                                     cpskip, cpoffset);
1078                                 if (bus_dmamap_create(sc->sc_dmat, 0xfff0,
1079                                     UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT,
1080                                     &q->q_dst_map) != 0) {
1081                                         err = ENOMEM;
1082                                         goto errout;
1083                                 }
1084                                 if (bus_dmamap_load_mbuf(sc->sc_dmat,
1085                                     q->q_dst_map, q->q_dst_m,
1086                                     BUS_DMA_NOWAIT) != 0) {
1087                                         bus_dmamap_destroy(sc->sc_dmat,
1088                                         q->q_dst_map);
1089                                         q->q_dst_map = NULL;
1090                                         err = ENOMEM;
1091                                         goto errout;
1092                                 }
1093                         }
1094                 } else {
1095                         err = EINVAL;
1096                         goto errout;
1097                 }
1098
1099 #ifdef UBSEC_DEBUG
1100                 printf("dst skip: %d\n", dskip);
1101 #endif
1102                 for (i = j = 0; i < q->q_dst_map->dm_nsegs; i++) {
1103                         struct ubsec_pktbuf *pb;
1104                         bus_size_t packl = q->q_dst_map->dm_segs[i].ds_len;
1105                         bus_addr_t packp = q->q_dst_map->dm_segs[i].ds_addr;
1106
1107                         if (dskip >= packl) {
1108                                 dskip -= packl;
1109                                 continue;
1110                         }
1111
1112                         packl -= dskip;
1113                         packp += dskip;
1114                         dskip = 0;
1115
1116                         if (packl > 0xfffc) {
1117                                 err = EIO;
1118                                 goto errout;
1119                         }
1120
1121                         if (j == 0)
1122                                 pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
1123                         else
1124                                 pb = &dmap->d_dma->d_dbuf[j - 1];
1125
1126                         pb->pb_addr = htole32(packp);
1127
1128                         if (dtheend) {
1129                                 if (packl > dtheend) {
1130                                         pb->pb_len = htole32(dtheend);
1131                                         dtheend = 0;
1132                                 } else {
1133                                         pb->pb_len = htole32(packl);
1134                                         dtheend -= packl;
1135                                 }
1136                         } else
1137                                 pb->pb_len = htole32(packl);
1138
1139                         if ((i + 1) == q->q_dst_map->dm_nsegs) {
1140                                 if (maccrd)
1141                                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1142                                             offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1143                                 else
1144                                         pb->pb_next = 0;
1145                         } else
1146                                 pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1147                                     offsetof(struct ubsec_dmachunk, d_dbuf[j]));
1148                         j++;
1149                 }
1150         }
1151
1152         dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
1153             offsetof(struct ubsec_dmachunk, d_ctx));
1154
1155         if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
1156                 struct ubsec_pktctx_long *ctxl;
1157
1158                 ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr +
1159                     offsetof(struct ubsec_dmachunk, d_ctx));
1160
1161                 /* transform small context into long context */
1162                 ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long));
1163                 ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC);
1164                 ctxl->pc_flags = ctx.pc_flags;
1165                 ctxl->pc_offset = ctx.pc_offset;
1166                 for (i = 0; i < 6; i++)
1167                         ctxl->pc_deskey[i] = ctx.pc_deskey[i];
1168                 for (i = 0; i < 5; i++)
1169                         ctxl->pc_hminner[i] = ctx.pc_hminner[i];
1170                 for (i = 0; i < 5; i++)
1171                         ctxl->pc_hmouter[i] = ctx.pc_hmouter[i];
1172                 ctxl->pc_iv[0] = ctx.pc_iv[0];
1173                 ctxl->pc_iv[1] = ctx.pc_iv[1];
1174         } else
1175                 bcopy(&ctx, dmap->d_alloc.dma_vaddr +
1176                     offsetof(struct ubsec_dmachunk, d_ctx),
1177                     sizeof(struct ubsec_pktctx));
1178
1179         s = splnet();
1180         SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
1181         sc->sc_nqueue++;
1182         ubsecstats.hst_ipackets++;
1183         ubsecstats.hst_ibytes += dmap->d_alloc.dma_map->dm_mapsize;
1184         ubsec_feed(sc);
1185         splx(s);
1186         return (0);
1187
1188 errout:
1189         if (q != NULL) {
1190                 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
1191                         m_freem(q->q_dst_m);
1192
1193                 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1194                         bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1195                         bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1196                 }
1197                 if (q->q_src_map != NULL) {
1198                         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1199                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1200                 }
1201
1202                 s = splnet();
1203                 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1204                 splx(s);
1205         }
1206         if (err == EINVAL)
1207                 ubsecstats.hst_invalid++;
1208         else
1209                 ubsecstats.hst_nomem++;
1210 errout2:
1211         crp->crp_etype = err;
1212         crypto_done(crp);
1213         return (0);
1214 }
1215
1216 void
1217 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
1218 {
1219         struct cryptop *crp = (struct cryptop *)q->q_crp;
1220         struct cryptodesc *crd;
1221         struct ubsec_dma *dmap = q->q_dma;
1222
1223         ubsecstats.hst_opackets++;
1224         ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
1225
1226         bus_dmamap_sync(sc->sc_dmat, dmap->d_alloc.dma_map, 0,
1227             dmap->d_alloc.dma_map->dm_mapsize,
1228             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
1229         if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1230                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
1231                     0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
1232                 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1233                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1234         }
1235         bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
1236             0, q->q_src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
1237         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1238         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1239
1240         if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
1241                 m_freem(q->q_src_m);
1242                 crp->crp_buf = (caddr_t)q->q_dst_m;
1243         }
1244
1245         /* copy out IV for future use */
1246         if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
1247                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1248                         if (crd->crd_alg != CRYPTO_DES_CBC &&
1249                             crd->crd_alg != CRYPTO_3DES_CBC)
1250                                 continue;
1251                         if (crp->crp_flags & CRYPTO_F_IMBUF)
1252                                 m_copydata((struct mbuf *)crp->crp_buf,
1253                                     crd->crd_skip + crd->crd_len - 8, 8,
1254                                     (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1255                         else if (crp->crp_flags & CRYPTO_F_IOV) {
1256                                 cuio_copydata((struct uio *)crp->crp_buf,
1257                                     crd->crd_skip + crd->crd_len - 8, 8,
1258                                     (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1259                         }
1260                         break;
1261                 }
1262         }
1263
1264         for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1265                 if (crd->crd_alg != CRYPTO_MD5_HMAC &&
1266                     crd->crd_alg != CRYPTO_SHA1_HMAC)
1267                         continue;
1268                 if (crp->crp_flags & CRYPTO_F_IMBUF)
1269                         m_copyback((struct mbuf *)crp->crp_buf,
1270                             crd->crd_inject, 12,
1271                             dmap->d_dma->d_macbuf);
1272                 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac)
1273                         bcopy((caddr_t)dmap->d_dma->d_macbuf,
1274                             crp->crp_mac, 12);
1275                 break;
1276         }
1277         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1278         crypto_done(crp);
1279 }
1280
1281 void
1282 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset)
1283 {
1284         int i, j, dlen, slen;
1285         caddr_t dptr, sptr;
1286
1287         j = 0;
1288         sptr = srcm->m_data;
1289         slen = srcm->m_len;
1290         dptr = dstm->m_data;
1291         dlen = dstm->m_len;
1292
1293         while (1) {
1294                 for (i = 0; i < min(slen, dlen); i++) {
1295                         if (j < hoffset || j >= toffset)
1296                                 *dptr++ = *sptr++;
1297                         slen--;
1298                         dlen--;
1299                         j++;
1300                 }
1301                 if (slen == 0) {
1302                         srcm = srcm->m_next;
1303                         if (srcm == NULL)
1304                                 return;
1305                         sptr = srcm->m_data;
1306                         slen = srcm->m_len;
1307                 }
1308                 if (dlen == 0) {
1309                         dstm = dstm->m_next;
1310                         if (dstm == NULL)
1311                                 return;
1312                         dptr = dstm->m_data;
1313                         dlen = dstm->m_len;
1314                 }
1315         }
1316 }
1317
1318 /*
1319  * feed the key generator, must be called at splnet() or higher.
1320  */
1321 void
1322 ubsec_feed2(struct ubsec_softc *sc)
1323 {
1324         struct ubsec_q2 *q;
1325
1326         while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
1327                 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
1328                         break;
1329                 q = SIMPLEQ_FIRST(&sc->sc_queue2);
1330
1331                 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0,
1332                     q->q_mcr.dma_map->dm_mapsize,
1333                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1334                 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0,
1335                     q->q_ctx.dma_map->dm_mapsize,
1336                     BUS_DMASYNC_PREWRITE);
1337
1338                 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
1339                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q_next);
1340                 --sc->sc_nqueue2;
1341                 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
1342         }
1343 }
1344
1345 /*
1346  * Callback for handling random numbers
1347  */
1348 void
1349 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q)
1350 {
1351         struct cryptkop *krp;
1352         struct ubsec_ctx_keyop *ctx;
1353
1354         ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
1355         bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0,
1356             q->q_ctx.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
1357
1358         switch (q->q_type) {
1359 #ifndef UBSEC_NO_RNG
1360         case UBS_CTXOP_RNGSHA1:
1361         case UBS_CTXOP_RNGBYPASS: {
1362                 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q;
1363                 u_int32_t *p;
1364                 int i;
1365
1366                 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0,
1367                     rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
1368                 p = (u_int32_t *)rng->rng_buf.dma_vaddr;
1369                 for (i = 0; i < UBSEC_RNG_BUFSIZ; p++, i++)
1370                         add_true_randomness(*p);
1371                 rng->rng_used = 0;
1372                 timeout_add(&sc->sc_rngto, sc->sc_rnghz);
1373                 break;
1374         }
1375 #endif
1376         case UBS_CTXOP_MODEXP: {
1377                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
1378                 u_int rlen, clen;
1379
1380                 krp = me->me_krp;
1381                 rlen = (me->me_modbits + 7) / 8;
1382                 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8;
1383
1384                 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map,
1385                     0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
1386                 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map,
1387                     0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
1388                 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map,
1389                     0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
1390                 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map,
1391                     0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
1392
1393                 if (clen < rlen)
1394                         krp->krp_status = E2BIG;
1395                 else {
1396                         if (sc->sc_flags & UBS_FLAGS_HWNORM) {
1397                                 bzero(krp->krp_param[krp->krp_iparams].crp_p,
1398                                     (krp->krp_param[krp->krp_iparams].crp_nbits
1399                                         + 7) / 8);
1400                                 bcopy(me->me_C.dma_vaddr,
1401                                     krp->krp_param[krp->krp_iparams].crp_p,
1402                                     (me->me_modbits + 7) / 8);
1403                         } else
1404                                 ubsec_kshift_l(me->me_shiftbits,
1405                                     me->me_C.dma_vaddr, me->me_normbits,
1406                                     krp->krp_param[krp->krp_iparams].crp_p,
1407                                     krp->krp_param[krp->krp_iparams].crp_nbits);
1408                 }
1409                 crypto_kdone(krp);
1410
1411                 /* bzero all potentially sensitive data */
1412                 bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
1413                 bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
1414                 bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
1415                 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
1416
1417                 /* Can't free here, so put us on the free list. */
1418                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
1419                 break;
1420         }
1421         case UBS_CTXOP_RSAPRIV: {
1422                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
1423                 u_int len;
1424
1425                 krp = rp->rpr_krp;
1426                 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 0,
1427                     rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
1428                 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 0,
1429                     rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
1430
1431                 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8;
1432                 bcopy(rp->rpr_msgout.dma_vaddr,
1433                     krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
1434
1435                 crypto_kdone(krp);
1436
1437                 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
1438                 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
1439                 bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size);
1440
1441                 /* Can't free here, so put us on the free list. */
1442                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
1443                 break;
1444         }
1445         default:
1446                 printf("%s: unknown ctx op: %x\n", sc->sc_dv.dv_xname,
1447                     letoh16(ctx->ctx_op));
1448                 break;
1449         }
1450 }
1451
1452 #ifndef UBSEC_NO_RNG
1453 void
1454 ubsec_rng(void *vsc)
1455 {
1456         struct ubsec_softc *sc = vsc;
1457         struct ubsec_q2_rng *rng = &sc->sc_rng;
1458         struct ubsec_mcr *mcr;
1459         struct ubsec_ctx_rngbypass *ctx;
1460         int s;
1461
1462         s = splnet();
1463         if (rng->rng_used) {
1464                 splx(s);
1465                 return;
1466         }
1467         sc->sc_nqueue2++;
1468         if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE)
1469                 goto out;
1470
1471         mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
1472         ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
1473
1474         mcr->mcr_pkts = htole16(1);
1475         mcr->mcr_flags = 0;
1476         mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
1477         mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0;
1478         mcr->mcr_ipktbuf.pb_len = 0;
1479         mcr->mcr_reserved = mcr->mcr_pktlen = 0;
1480         mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
1481         mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
1482             UBS_PKTBUF_LEN);
1483         mcr->mcr_opktbuf.pb_next = 0;
1484
1485         ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
1486         ctx->rbp_op = htole16(UBS_CTXOP_RNGSHA1);
1487         rng->rng_q.q_type = UBS_CTXOP_RNGSHA1;
1488
1489         bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0,
1490             rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
1491
1492         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
1493         rng->rng_used = 1;
1494         ubsec_feed2(sc);
1495         splx(s);
1496
1497         return;
1498
1499 out:
1500         /*
1501          * Something weird happened, generate our own call back.
1502          */
1503         sc->sc_nqueue2--;
1504         splx(s);
1505         timeout_add(&sc->sc_rngto, sc->sc_rnghz);
1506 }
1507 #endif /* UBSEC_NO_RNG */
1508
1509 int
1510 ubsec_dma_malloc(struct ubsec_softc *sc, bus_size_t size,
1511     struct ubsec_dma_alloc *dma, int mapflags)
1512 {
1513         int r;
1514
1515         if ((r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0,
1516             &dma->dma_seg, 1, &dma->dma_nseg, BUS_DMA_NOWAIT)) != 0)
1517                 goto fail_0;
1518
1519         if ((r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg,
1520             size, &dma->dma_vaddr, mapflags | BUS_DMA_NOWAIT)) != 0)
1521                 goto fail_1;
1522
1523         if ((r = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
1524             BUS_DMA_NOWAIT, &dma->dma_map)) != 0)
1525                 goto fail_2;
1526
1527         if ((r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr,
1528             size, NULL, BUS_DMA_NOWAIT)) != 0)
1529                 goto fail_3;
1530
1531         dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr;
1532         dma->dma_size = size;
1533         return (0);
1534
1535 fail_3:
1536         bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
1537 fail_2:
1538         bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size);
1539 fail_1:
1540         bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
1541 fail_0:
1542         dma->dma_map = NULL;
1543         return (r);
1544 }
1545
1546 void
1547 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma)
1548 {
1549         bus_dmamap_unload(sc->sc_dmat, dma->dma_map);
1550         bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size);
1551         bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
1552         bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
1553 }
1554
1555 /*
1556  * Resets the board.  Values in the regesters are left as is
1557  * from the reset (i.e. initial values are assigned elsewhere).
1558  */
1559 void
1560 ubsec_reset_board(struct ubsec_softc *sc)
1561 {
1562     volatile u_int32_t ctrl;
1563
1564     ctrl = READ_REG(sc, BS_CTRL);
1565     ctrl |= BS_CTRL_RESET;
1566     WRITE_REG(sc, BS_CTRL, ctrl);
1567
1568     /*
1569      * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
1570      */
1571     DELAY(10);
1572 }
1573
1574 /*
1575  * Init Broadcom registers
1576  */
1577 void
1578 ubsec_init_board(struct ubsec_softc *sc)
1579 {
1580         u_int32_t ctrl;
1581
1582         ctrl = READ_REG(sc, BS_CTRL);
1583         ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64);
1584         ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT;
1585
1586         if (sc->sc_flags & UBS_FLAGS_KEY)
1587                 ctrl |= BS_CTRL_MCR2INT;
1588         else
1589                 ctrl &= ~BS_CTRL_MCR2INT;
1590
1591         if (sc->sc_flags & UBS_FLAGS_HWNORM)
1592                 ctrl &= ~BS_CTRL_SWNORM;
1593
1594         WRITE_REG(sc, BS_CTRL, ctrl);
1595 }
1596
1597 /*
1598  * Init Broadcom PCI registers
1599  */
1600 void
1601 ubsec_init_pciregs(struct pci_attach_args *pa)
1602 {
1603         pci_chipset_tag_t pc = pa->pa_pc;
1604         u_int32_t misc;
1605
1606         /*
1607          * This will set the cache line size to 1, this will
1608          * force the BCM58xx chip just to do burst read/writes.
1609          * Cache line read/writes are to slow
1610          */
1611         misc = pci_conf_read(pc, pa->pa_tag, PCI_BHLC_REG);
1612         misc = (misc & ~(PCI_CACHELINE_MASK << PCI_CACHELINE_SHIFT))
1613             | ((UBS_DEF_CACHELINE & 0xff) << PCI_CACHELINE_SHIFT);
1614         pci_conf_write(pc, pa->pa_tag, PCI_BHLC_REG, misc);
1615 }
1616
1617 /*
1618  * Clean up after a chip crash.
1619  * It is assumed that the caller is in splnet()
1620  */
1621 void
1622 ubsec_cleanchip(struct ubsec_softc *sc)
1623 {
1624         struct ubsec_q *q;
1625
1626         while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
1627                 q = SIMPLEQ_FIRST(&sc->sc_qchip);
1628                 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
1629                 ubsec_free_q(sc, q);
1630         }
1631 }
1632
1633 /*
1634  * free a ubsec_q
1635  * It is assumed that the caller is within splnet()
1636  */
1637 int
1638 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
1639 {
1640         struct ubsec_q *q2;
1641         struct cryptop *crp;
1642         int npkts;
1643         int i;
1644
1645         npkts = q->q_nstacked_mcrs;
1646
1647         for (i = 0; i < npkts; i++) {
1648                 if(q->q_stacked_mcr[i]) {
1649                         q2 = q->q_stacked_mcr[i];
1650
1651                         if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
1652                                 m_freem(q2->q_dst_m);
1653
1654                         crp = (struct cryptop *)q2->q_crp;
1655
1656                         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
1657
1658                         crp->crp_etype = EFAULT;
1659                         crypto_done(crp);
1660                 } else {
1661                         break;
1662                 }
1663         }
1664
1665         /*
1666          * Free header MCR
1667          */
1668         if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
1669                 m_freem(q->q_dst_m);
1670
1671         crp = (struct cryptop *)q->q_crp;
1672
1673         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1674
1675         crp->crp_etype = EFAULT;
1676         crypto_done(crp);
1677         return(0);
1678 }
1679
1680 /*
1681  * Routine to reset the chip and clean up.
1682  * It is assumed that the caller is in splnet()
1683  */
1684 void
1685 ubsec_totalreset(struct ubsec_softc *sc)
1686 {
1687         ubsec_reset_board(sc);
1688         ubsec_init_board(sc);
1689         ubsec_cleanchip(sc);
1690 }
1691
1692 int
1693 ubsec_dmamap_aligned(bus_dmamap_t map)
1694 {
1695         int i;
1696
1697         for (i = 0; i < map->dm_nsegs; i++) {
1698                 if (map->dm_segs[i].ds_addr & 3)
1699                         return (0);
1700                 if ((i != (map->dm_nsegs - 1)) &&
1701                     (map->dm_segs[i].ds_len & 3))
1702                         return (0);
1703         }
1704         return (1);
1705 }
1706
1707 struct ubsec_softc *
1708 ubsec_kfind(struct cryptkop *krp)
1709 {
1710         struct ubsec_softc *sc;
1711         int i;
1712
1713         for (i = 0; i < ubsec_cd.cd_ndevs; i++) {
1714                 sc = ubsec_cd.cd_devs[i];
1715                 if (sc == NULL)
1716                         continue;
1717                 if (sc->sc_cid == krp->krp_hid)
1718                         return (sc);
1719         }
1720         return (NULL);
1721 }
1722
1723 void
1724 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q)
1725 {
1726         switch (q->q_type) {
1727         case UBS_CTXOP_MODEXP: {
1728                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
1729
1730                 ubsec_dma_free(sc, &me->me_q.q_mcr);
1731                 ubsec_dma_free(sc, &me->me_q.q_ctx);
1732                 ubsec_dma_free(sc, &me->me_M);
1733                 ubsec_dma_free(sc, &me->me_E);
1734                 ubsec_dma_free(sc, &me->me_C);
1735                 ubsec_dma_free(sc, &me->me_epb);
1736                 free(me, M_DEVBUF);
1737                 break;
1738         }
1739         case UBS_CTXOP_RSAPRIV: {
1740                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
1741
1742                 ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
1743                 ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
1744                 ubsec_dma_free(sc, &rp->rpr_msgin);
1745                 ubsec_dma_free(sc, &rp->rpr_msgout);
1746                 free(rp, M_DEVBUF);
1747                 break;
1748         }
1749         default:
1750                 printf("%s: invalid kfree 0x%x\n", sc->sc_dv.dv_xname,
1751                     q->q_type);
1752                 break;
1753         }
1754 }
1755
1756 int
1757 ubsec_kprocess(struct cryptkop *krp)
1758 {
1759         struct ubsec_softc *sc;
1760         int r;
1761
1762         if (krp == NULL || krp->krp_callback == NULL)
1763                 return (EINVAL);
1764         if ((sc = ubsec_kfind(krp)) == NULL)
1765                 return (EINVAL);
1766
1767         while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
1768                 struct ubsec_q2 *q;
1769
1770                 q = SIMPLEQ_FIRST(&sc->sc_q2free);
1771                 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q_next);
1772                 ubsec_kfree(sc, q);
1773         }
1774
1775         switch (krp->krp_op) {
1776         case CRK_MOD_EXP:
1777                 if (sc->sc_flags & UBS_FLAGS_HWNORM)
1778                         r = ubsec_kprocess_modexp_hw(sc, krp);
1779                 else
1780                         r = ubsec_kprocess_modexp_sw(sc, krp);
1781                 break;
1782         case CRK_MOD_EXP_CRT:
1783                 r = ubsec_kprocess_rsapriv(sc, krp);
1784                 break;
1785         default:
1786                 printf("%s: kprocess: invalid op 0x%x\n",
1787                     sc->sc_dv.dv_xname, krp->krp_op);
1788                 krp->krp_status = EOPNOTSUPP;
1789                 crypto_kdone(krp);
1790                 r = 0;
1791         }
1792         return (r);
1793 }
1794
1795 /*
1796  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
1797  */
1798 int
1799 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp)
1800 {
1801         struct ubsec_q2_modexp *me;
1802         struct ubsec_mcr *mcr;
1803         struct ubsec_ctx_modexp *ctx;
1804         struct ubsec_pktbuf *epb;
1805         int err = 0, s;
1806         u_int nbits, normbits, mbits, shiftbits, ebits;
1807
1808         me = malloc(sizeof *me, M_DEVBUF, M_NOWAIT | M_ZERO);
1809         if (me == NULL) {
1810                 err = ENOMEM;
1811                 goto errout;
1812         }
1813         me->me_krp = krp;
1814         me->me_q.q_type = UBS_CTXOP_MODEXP;
1815
1816         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
1817         if (nbits <= 512)
1818                 normbits = 512;
1819         else if (nbits <= 768)
1820                 normbits = 768;
1821         else if (nbits <= 1024)
1822                 normbits = 1024;
1823         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
1824                 normbits = 1536;
1825         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
1826                 normbits = 2048;
1827         else {
1828                 err = E2BIG;
1829                 goto errout;
1830         }
1831
1832         shiftbits = normbits - nbits;
1833
1834         me->me_modbits = nbits;
1835         me->me_shiftbits = shiftbits;
1836         me->me_normbits = normbits;
1837
1838         /* Sanity check: result bits must be >= true modulus bits. */
1839         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
1840                 err = ERANGE;
1841                 goto errout;
1842         }
1843
1844         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
1845             &me->me_q.q_mcr, 0)) {
1846                 err = ENOMEM;
1847                 goto errout;
1848         }
1849         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
1850
1851         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
1852             &me->me_q.q_ctx, 0)) {
1853                 err = ENOMEM;
1854                 goto errout;
1855         }
1856
1857         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
1858         if (mbits > nbits) {
1859                 err = E2BIG;
1860                 goto errout;
1861         }
1862         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
1863                 err = ENOMEM;
1864                 goto errout;
1865         }
1866         ubsec_kshift_r(shiftbits,
1867             krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
1868             me->me_M.dma_vaddr, normbits);
1869
1870         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
1871                 err = ENOMEM;
1872                 goto errout;
1873         }
1874         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
1875
1876         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
1877         if (ebits > nbits) {
1878                 err = E2BIG;
1879                 goto errout;
1880         }
1881         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
1882                 err = ENOMEM;
1883                 goto errout;
1884         }
1885         ubsec_kshift_r(shiftbits,
1886             krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
1887             me->me_E.dma_vaddr, normbits);
1888
1889         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
1890             &me->me_epb, 0)) {
1891                 err = ENOMEM;
1892                 goto errout;
1893         }
1894         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
1895         epb->pb_addr = htole32(me->me_E.dma_paddr);
1896         epb->pb_next = 0;
1897         epb->pb_len = htole32(normbits / 8);
1898
1899 #ifdef UBSEC_DEBUG
1900         printf("Epb ");
1901         ubsec_dump_pb(epb);
1902 #endif
1903
1904         mcr->mcr_pkts = htole16(1);
1905         mcr->mcr_flags = 0;
1906         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
1907         mcr->mcr_reserved = 0;
1908         mcr->mcr_pktlen = 0;
1909
1910         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
1911         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
1912         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
1913
1914         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
1915         mcr->mcr_opktbuf.pb_next = 0;
1916         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
1917
1918 #ifdef DIAGNOSTIC
1919         /* Misaligned output buffer will hang the chip. */
1920         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
1921                 panic("%s: modexp invalid addr 0x%x",
1922                     sc->sc_dv.dv_xname, letoh32(mcr->mcr_opktbuf.pb_addr));
1923         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
1924                 panic("%s: modexp invalid len 0x%x",
1925                     sc->sc_dv.dv_xname, letoh32(mcr->mcr_opktbuf.pb_len));
1926 #endif
1927
1928         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
1929         bzero(ctx, sizeof(*ctx));
1930         ubsec_kshift_r(shiftbits,
1931             krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
1932             ctx->me_N, normbits);
1933         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
1934         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
1935         ctx->me_E_len = htole16(nbits);
1936         ctx->me_N_len = htole16(nbits);
1937
1938 #ifdef UBSEC_DEBUG
1939         ubsec_dump_mcr(mcr);
1940         ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
1941 #endif
1942
1943         /*
1944          * ubsec_feed2 will sync mcr and ctx, we just need to sync
1945          * everything else.
1946          */
1947         bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map,
1948             0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
1949         bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map,
1950             0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
1951         bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map,
1952             0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
1953         bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map,
1954             0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
1955
1956         /* Enqueue and we're done... */
1957         s = splnet();
1958         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
1959         ubsec_feed2(sc);
1960         splx(s);
1961
1962         return (0);
1963
1964 errout:
1965         if (me != NULL) {
1966                 if (me->me_q.q_mcr.dma_map != NULL)
1967                         ubsec_dma_free(sc, &me->me_q.q_mcr);
1968                 if (me->me_q.q_ctx.dma_map != NULL) {
1969                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
1970                         ubsec_dma_free(sc, &me->me_q.q_ctx);
1971                 }
1972                 if (me->me_M.dma_map != NULL) {
1973                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
1974                         ubsec_dma_free(sc, &me->me_M);
1975                 }
1976                 if (me->me_E.dma_map != NULL) {
1977                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
1978                         ubsec_dma_free(sc, &me->me_E);
1979                 }
1980                 if (me->me_C.dma_map != NULL) {
1981                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
1982                         ubsec_dma_free(sc, &me->me_C);
1983                 }
1984                 if (me->me_epb.dma_map != NULL)
1985                         ubsec_dma_free(sc, &me->me_epb);
1986                 free(me, M_DEVBUF);
1987         }
1988         krp->krp_status = err;
1989         crypto_kdone(krp);
1990         return (0);
1991 }
1992
1993 /*
1994  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
1995  */
1996 int
1997 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp)
1998 {
1999         struct ubsec_q2_modexp *me;
2000         struct ubsec_mcr *mcr;
2001         struct ubsec_ctx_modexp *ctx;
2002         struct ubsec_pktbuf *epb;
2003         int err = 0, s;
2004         u_int nbits, normbits, mbits, shiftbits, ebits;
2005
2006         me = malloc(sizeof *me, M_DEVBUF, M_NOWAIT | M_ZERO);
2007         if (me == NULL) {
2008                 err = ENOMEM;
2009                 goto errout;
2010         }
2011         me->me_krp = krp;
2012         me->me_q.q_type = UBS_CTXOP_MODEXP;
2013
2014         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2015         if (nbits <= 512)
2016                 normbits = 512;
2017         else if (nbits <= 768)
2018                 normbits = 768;
2019         else if (nbits <= 1024)
2020                 normbits = 1024;
2021         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2022                 normbits = 1536;
2023         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2024                 normbits = 2048;
2025         else {
2026                 err = E2BIG;
2027                 goto errout;
2028         }
2029
2030         shiftbits = normbits - nbits;
2031
2032         /* XXX ??? */
2033         me->me_modbits = nbits;
2034         me->me_shiftbits = shiftbits;
2035         me->me_normbits = normbits;
2036
2037         /* Sanity check: result bits must be >= true modulus bits. */
2038         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2039                 err = ERANGE;
2040                 goto errout;
2041         }
2042
2043         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2044             &me->me_q.q_mcr, 0)) {
2045                 err = ENOMEM;
2046                 goto errout;
2047         }
2048         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2049
2050         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2051             &me->me_q.q_ctx, 0)) {
2052                 err = ENOMEM;
2053                 goto errout;
2054         }
2055
2056         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2057         if (mbits > nbits) {
2058                 err = E2BIG;
2059                 goto errout;
2060         }
2061         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2062                 err = ENOMEM;
2063                 goto errout;
2064         }
2065         bzero(me->me_M.dma_vaddr, normbits / 8);
2066         bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
2067             me->me_M.dma_vaddr, (mbits + 7) / 8);
2068
2069         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2070                 err = ENOMEM;
2071                 goto errout;
2072         }
2073         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2074
2075         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2076         if (ebits > nbits) {
2077                 err = E2BIG;
2078                 goto errout;
2079         }
2080         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2081                 err = ENOMEM;
2082                 goto errout;
2083         }
2084         bzero(me->me_E.dma_vaddr, normbits / 8);
2085         bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
2086             me->me_E.dma_vaddr, (ebits + 7) / 8);
2087
2088         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2089             &me->me_epb, 0)) {
2090                 err = ENOMEM;
2091                 goto errout;
2092         }
2093         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2094         epb->pb_addr = htole32(me->me_E.dma_paddr);
2095         epb->pb_next = 0;
2096         epb->pb_len = htole32((ebits + 7) / 8);
2097
2098 #ifdef UBSEC_DEBUG
2099         printf("Epb ");
2100         ubsec_dump_pb(epb);
2101 #endif
2102
2103         mcr->mcr_pkts = htole16(1);
2104         mcr->mcr_flags = 0;
2105         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2106         mcr->mcr_reserved = 0;
2107         mcr->mcr_pktlen = 0;
2108
2109         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2110         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2111         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2112
2113         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2114         mcr->mcr_opktbuf.pb_next = 0;
2115         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2116
2117 #ifdef DIAGNOSTIC
2118         /* Misaligned output buffer will hang the chip. */
2119         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2120                 panic("%s: modexp invalid addr 0x%x",
2121                     sc->sc_dv.dv_xname, letoh32(mcr->mcr_opktbuf.pb_addr));
2122         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2123                 panic("%s: modexp invalid len 0x%x",
2124                     sc->sc_dv.dv_xname, letoh32(mcr->mcr_opktbuf.pb_len));
2125 #endif
2126
2127         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2128         bzero(ctx, sizeof(*ctx));
2129         bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N,
2130             (nbits + 7) / 8);
2131         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2132         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2133         ctx->me_E_len = htole16(ebits);
2134         ctx->me_N_len = htole16(nbits);
2135
2136 #ifdef UBSEC_DEBUG
2137         ubsec_dump_mcr(mcr);
2138         ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2139 #endif
2140
2141         /*
2142          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2143          * everything else.
2144          */
2145         bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map,
2146             0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2147         bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map,
2148             0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2149         bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map,
2150             0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2151         bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map,
2152             0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2153
2154         /* Enqueue and we're done... */
2155         s = splnet();
2156         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2157         ubsec_feed2(sc);
2158         splx(s);
2159
2160         return (0);
2161
2162 errout:
2163         if (me != NULL) {
2164                 if (me->me_q.q_mcr.dma_map != NULL)
2165                         ubsec_dma_free(sc, &me->me_q.q_mcr);
2166                 if (me->me_q.q_ctx.dma_map != NULL) {
2167                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
2168                         ubsec_dma_free(sc, &me->me_q.q_ctx);
2169                 }
2170                 if (me->me_M.dma_map != NULL) {
2171                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
2172                         ubsec_dma_free(sc, &me->me_M);
2173                 }
2174                 if (me->me_E.dma_map != NULL) {
2175                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
2176                         ubsec_dma_free(sc, &me->me_E);
2177                 }
2178                 if (me->me_C.dma_map != NULL) {
2179                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2180                         ubsec_dma_free(sc, &me->me_C);
2181                 }
2182                 if (me->me_epb.dma_map != NULL)
2183                         ubsec_dma_free(sc, &me->me_epb);
2184                 free(me, M_DEVBUF);
2185         }
2186         krp->krp_status = err;
2187         crypto_kdone(krp);
2188         return (0);
2189 }
2190
2191 int
2192 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp)
2193 {
2194         struct ubsec_q2_rsapriv *rp = NULL;
2195         struct ubsec_mcr *mcr;
2196         struct ubsec_ctx_rsapriv *ctx;
2197         int err = 0, s;
2198         u_int padlen, msglen;
2199
2200         msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
2201         padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
2202         if (msglen > padlen)
2203                 padlen = msglen;
2204
2205         if (padlen <= 256)
2206                 padlen = 256;
2207         else if (padlen <= 384)
2208                 padlen = 384;
2209         else if (padlen <= 512)
2210                 padlen = 512;
2211         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768)
2212                 padlen = 768;
2213         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024)
2214                 padlen = 1024;
2215         else {
2216                 err = E2BIG;
2217                 goto errout;
2218         }
2219
2220         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
2221                 err = E2BIG;
2222                 goto errout;
2223         }
2224
2225         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
2226                 err = E2BIG;
2227                 goto errout;
2228         }
2229
2230         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
2231                 err = E2BIG;
2232                 goto errout;
2233         }
2234
2235         rp = malloc(sizeof *rp, M_DEVBUF, M_NOWAIT | M_ZERO);
2236         if (rp == NULL)
2237                 return (ENOMEM);
2238         rp->rpr_krp = krp;
2239         rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
2240
2241         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2242             &rp->rpr_q.q_mcr, 0)) {
2243                 err = ENOMEM;
2244                 goto errout;
2245         }
2246         mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
2247
2248         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
2249             &rp->rpr_q.q_ctx, 0)) {
2250                 err = ENOMEM;
2251                 goto errout;
2252         }
2253         ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
2254         bzero(ctx, sizeof *ctx);
2255
2256         /* Copy in p */
2257         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
2258             &ctx->rpr_buf[0 * (padlen / 8)],
2259             (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8);
2260
2261         /* Copy in q */
2262         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
2263             &ctx->rpr_buf[1 * (padlen / 8)],
2264             (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8);
2265
2266         /* Copy in dp */
2267         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
2268             &ctx->rpr_buf[2 * (padlen / 8)],
2269             (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8);
2270
2271         /* Copy in dq */
2272         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
2273             &ctx->rpr_buf[3 * (padlen / 8)],
2274             (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8);
2275
2276         /* Copy in pinv */
2277         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
2278             &ctx->rpr_buf[4 * (padlen / 8)],
2279             (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8);
2280
2281         msglen = padlen * 2;
2282
2283         /* Copy in input message (aligned buffer/length). */
2284         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
2285                 /* Is this likely? */
2286                 err = E2BIG;
2287                 goto errout;
2288         }
2289         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) {
2290                 err = ENOMEM;
2291                 goto errout;
2292         }
2293         bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8);
2294         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
2295             rp->rpr_msgin.dma_vaddr,
2296             (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8);
2297
2298         /* Prepare space for output message (aligned buffer/length). */
2299         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
2300                 /* Is this likely? */
2301                 err = E2BIG;
2302                 goto errout;
2303         }
2304         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) {
2305                 err = ENOMEM;
2306                 goto errout;
2307         }
2308         bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8);
2309
2310         mcr->mcr_pkts = htole16(1);
2311         mcr->mcr_flags = 0;
2312         mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
2313         mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
2314         mcr->mcr_ipktbuf.pb_next = 0;
2315         mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
2316         mcr->mcr_reserved = 0;
2317         mcr->mcr_pktlen = htole16(msglen);
2318         mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
2319         mcr->mcr_opktbuf.pb_next = 0;
2320         mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
2321
2322 #ifdef DIAGNOSTIC
2323         if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) {
2324                 panic("%s: rsapriv: invalid msgin %p(0x%x)",
2325                     sc->sc_dv.dv_xname, rp->rpr_msgin.dma_paddr,
2326                     rp->rpr_msgin.dma_size);
2327         }
2328         if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) {
2329                 panic("%s: rsapriv: invalid msgout %p(0x%x)",
2330                     sc->sc_dv.dv_xname, rp->rpr_msgout.dma_paddr,
2331                     rp->rpr_msgout.dma_size);
2332         }
2333 #endif
2334
2335         ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8));
2336         ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
2337         ctx->rpr_q_len = htole16(padlen);
2338         ctx->rpr_p_len = htole16(padlen);
2339
2340         /*
2341          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2342          * everything else.
2343          */
2344         bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map,
2345             0, rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2346         bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map,
2347             0, rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2348
2349         /* Enqueue and we're done... */
2350         s = splnet();
2351         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
2352         ubsec_feed2(sc);
2353         splx(s);
2354         return (0);
2355
2356 errout:
2357         if (rp != NULL) {
2358                 if (rp->rpr_q.q_mcr.dma_map != NULL)
2359                         ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2360                 if (rp->rpr_msgin.dma_map != NULL) {
2361                         bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
2362                         ubsec_dma_free(sc, &rp->rpr_msgin);
2363                 }
2364                 if (rp->rpr_msgout.dma_map != NULL) {
2365                         bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
2366                         ubsec_dma_free(sc, &rp->rpr_msgout);
2367                 }
2368                 free(rp, M_DEVBUF);
2369         }
2370         krp->krp_status = err;
2371         crypto_kdone(krp);
2372         return (0);
2373 }
2374
2375 void
2376 ubsec_dump_pb(struct ubsec_pktbuf *pb)
2377 {
2378         printf("addr 0x%x (0x%x) next 0x%x\n",
2379             pb->pb_addr, pb->pb_len, pb->pb_next);
2380 }
2381
2382 void
2383 ubsec_dump_ctx2(struct ubsec_ctx_keyop *c)
2384 {
2385         printf("CTX (0x%x):\n", c->ctx_len);
2386         switch (letoh16(c->ctx_op)) {
2387         case UBS_CTXOP_RNGBYPASS:
2388         case UBS_CTXOP_RNGSHA1:
2389                 break;
2390         case UBS_CTXOP_MODEXP:
2391         {
2392                 struct ubsec_ctx_modexp *cx = (void *)c;
2393                 int i, len;
2394
2395                 printf(" Elen %u, Nlen %u\n",
2396                     letoh16(cx->me_E_len), letoh16(cx->me_N_len));
2397                 len = (cx->me_N_len + 7)/8;
2398                 for (i = 0; i < len; i++)
2399                         printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]);
2400                 printf("\n");
2401                 break;
2402         }
2403         default:
2404                 printf("unknown context: %x\n", c->ctx_op);
2405         }
2406         printf("END CTX\n");
2407 }
2408
2409 void
2410 ubsec_dump_mcr(struct ubsec_mcr *mcr)
2411 {
2412         struct ubsec_mcr_add *ma;
2413         int i;
2414
2415         printf("MCR:\n");
2416         printf(" pkts: %u, flags 0x%x\n",
2417             letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
2418         ma = (struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
2419         for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
2420                 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
2421                     letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
2422                     letoh16(ma->mcr_reserved));
2423                 printf(" %d: ipkt ", i);
2424                 ubsec_dump_pb(&ma->mcr_ipktbuf);
2425                 printf(" %d: opkt ", i);
2426                 ubsec_dump_pb(&ma->mcr_opktbuf);
2427                 ma++;
2428         }
2429         printf("END MCR\n");
2430 }
2431
2432 /*
2433  * Return the number of significant bits of a big number.
2434  */
2435 int
2436 ubsec_ksigbits(struct crparam *cr)
2437 {
2438         u_int plen = (cr->crp_nbits + 7) / 8;
2439         int i, sig = plen * 8;
2440         u_int8_t c, *p = cr->crp_p;
2441
2442         for (i = plen - 1; i >= 0; i--) {
2443                 c = p[i];
2444                 if (c != 0) {
2445                         while ((c & 0x80) == 0) {
2446                                 sig--;
2447                                 c <<= 1;
2448                         }
2449                         break;
2450                 }
2451                 sig -= 8;
2452         }
2453         return (sig);
2454 }
2455
2456 void
2457 ubsec_kshift_r(u_int shiftbits, u_int8_t *src, u_int srcbits,
2458     u_int8_t *dst, u_int dstbits)
2459 {
2460         u_int slen, dlen;
2461         int i, si, di, n;
2462
2463         slen = (srcbits + 7) / 8;
2464         dlen = (dstbits + 7) / 8;
2465
2466         for (i = 0; i < slen; i++)
2467                 dst[i] = src[i];
2468         for (i = 0; i < dlen - slen; i++)
2469                 dst[slen + i] = 0;
2470
2471         n = shiftbits / 8;
2472         if (n != 0) {
2473                 si = dlen - n - 1;
2474                 di = dlen - 1;
2475                 while (si >= 0)
2476                         dst[di--] = dst[si--];
2477                 while (di >= 0)
2478                         dst[di--] = 0;
2479         }
2480
2481         n = shiftbits % 8;
2482         if (n != 0) {
2483                 for (i = dlen - 1; i > 0; i--)
2484                         dst[i] = (dst[i] << n) |
2485                             (dst[i - 1] >> (8 - n));
2486                 dst[0] = dst[0] << n;
2487         }
2488 }
2489
2490 void
2491 ubsec_kshift_l(u_int shiftbits, u_int8_t *src, u_int srcbits,
2492     u_int8_t *dst, u_int dstbits)
2493 {
2494         int slen, dlen, i, n;
2495
2496         slen = (srcbits + 7) / 8;
2497         dlen = (dstbits + 7) / 8;
2498
2499         n = shiftbits / 8;
2500         for (i = 0; i < slen; i++)
2501                 dst[i] = src[i + n];
2502         for (i = 0; i < dlen - slen; i++)
2503                 dst[slen + i] = 0;
2504
2505         n = shiftbits % 8;
2506         if (n != 0) {
2507                 for (i = 0; i < (dlen - 1); i++)
2508                         dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n));
2509                 dst[dlen - 1] = dst[dlen - 1] >> n;
2510         }
2511 }