Skip to content

Commit 24e8308

Browse files
raymondriscstarraymondriscstar
committed
core: riscv: add Linux vector SHA-256 and SHA-512 acceleration
Import RISC-V vector SHA-256 and SHA-512 compression routines from Linux v7.2, retaining their original filenames, dual Apache-2.0/BSD-2-Clause license notices, and upstream copyright attributions. Adapt only the Linux linkage macros to OP-TEE. Add OP-TEE crypto_drv glue and select each implementation independently with CFG_CORE_CRYPTO_SHA256_ACCEL and CFG_CORE_CRYPTO_SHA512_ACCEL. Signed-off-by: Raymond Mao <raymond.mao@riscstar.com>
1 parent a78428a commit 24e8308

6 files changed

Lines changed: 482 additions & 1 deletion

File tree

‎core/arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S‎

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
2+
//
3+
// This file is dual-licensed, meaning that you can use it under your
4+
// choice of either of the following two licenses:
5+
//
6+
// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
7+
//
8+
// Licensed under the Apache License 2.0 (the "License"). You can obtain
9+
// a copy in the file LICENSE in the source distribution or at
10+
// https://www.openssl.org/source/license.html
11+
//
12+
// or
13+
//
14+
// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu>
15+
// Copyright (c) 2023, Phoebe Chen <phoebe.chen@sifive.com>
16+
// Copyright 2024 Google LLC
17+
// All rights reserved.
18+
//
19+
// Redistribution and use in source and binary forms, with or without
20+
// modification, are permitted provided that the following conditions
21+
// are met:
22+
// 1. Redistributions of source code must retain the above copyright
23+
// notice, this list of conditions and the following disclaimer.
24+
// 2. Redistributions in binary form must reproduce the above copyright
25+
// notice, this list of conditions and the following disclaimer in the
26+
// documentation and/or other materials provided with the distribution.
27+
//
28+
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29+
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30+
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31+
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32+
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33+
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34+
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35+
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36+
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37+
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38+
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39+
40+
// The generated code of this file depends on the following RISC-V extensions:
41+
// - RV64I
42+
// - RISC-V Vector ('V') with VLEN >= 128
43+
// - RISC-V Vector SHA-2 Secure Hash extension ('Zvknha' or 'Zvknhb')
44+
// - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb')
45+
46+
#include <asm.S>
47+
48+
#define SYM_FUNC_START(name) FUNC name , :
49+
#define SYM_FUNC_END(name) END_FUNC name
50+
51+
.text
52+
.option arch, +zvknha, +zvkb
53+
54+
#define STATEP a0
55+
#define DATA a1
56+
#define NUM_BLOCKS a2
57+
58+
#define STATEP_C a3
59+
60+
#define MASK v0
61+
#define INDICES v1
62+
#define W0 v2
63+
#define W1 v3
64+
#define W2 v4
65+
#define W3 v5
66+
#define VTMP v6
67+
#define FEBA v7
68+
#define HGDC v8
69+
#define K0 v10
70+
#define K1 v11
71+
#define K2 v12
72+
#define K3 v13
73+
#define K4 v14
74+
#define K5 v15
75+
#define K6 v16
76+
#define K7 v17
77+
#define K8 v18
78+
#define K9 v19
79+
#define K10 v20
80+
#define K11 v21
81+
#define K12 v22
82+
#define K13 v23
83+
#define K14 v24
84+
#define K15 v25
85+
#define PREV_FEBA v26
86+
#define PREV_HGDC v27
87+
88+
// Do 4 rounds of SHA-256. w0 contains the current 4 message schedule words.
89+
//
90+
// If not all the message schedule words have been computed yet, then this also
91+
// computes 4 more message schedule words. w1-w3 contain the next 3 groups of 4
92+
// message schedule words; this macro computes the group after w3 and writes it
93+
// to w0. This means that the next (w0, w1, w2, w3) is the current (w1, w2, w3,
94+
// w0), so the caller must cycle through the registers accordingly.
95+
.macro sha256_4rounds last, k, w0, w1, w2, w3
96+
vadd.vv VTMP, \k, \w0
97+
vsha2cl.vv HGDC, FEBA, VTMP
98+
vsha2ch.vv FEBA, HGDC, VTMP
99+
.if !\last
100+
vmerge.vvm VTMP, \w2, \w1, MASK
101+
vsha2ms.vv \w0, VTMP, \w3
102+
.endif
103+
.endm
104+
105+
.macro sha256_16rounds last, k0, k1, k2, k3
106+
sha256_4rounds \last, \k0, W0, W1, W2, W3
107+
sha256_4rounds \last, \k1, W1, W2, W3, W0
108+
sha256_4rounds \last, \k2, W2, W3, W0, W1
109+
sha256_4rounds \last, \k3, W3, W0, W1, W2
110+
.endm
111+
112+
// void sha256_transform_zvknha_or_zvknhb_zvkb(struct sha256_block_state *state,
113+
// const u8 *data, size_t nblocks);
114+
SYM_FUNC_START(sha256_transform_zvknha_or_zvknhb_zvkb)
115+
116+
// Load the round constants into K0-K15.
117+
vsetivli zero, 4, e32, m1, ta, ma
118+
la t0, K256
119+
vle32.v K0, (t0)
120+
addi t0, t0, 16
121+
vle32.v K1, (t0)
122+
addi t0, t0, 16
123+
vle32.v K2, (t0)
124+
addi t0, t0, 16
125+
vle32.v K3, (t0)
126+
addi t0, t0, 16
127+
vle32.v K4, (t0)
128+
addi t0, t0, 16
129+
vle32.v K5, (t0)
130+
addi t0, t0, 16
131+
vle32.v K6, (t0)
132+
addi t0, t0, 16
133+
vle32.v K7, (t0)
134+
addi t0, t0, 16
135+
vle32.v K8, (t0)
136+
addi t0, t0, 16
137+
vle32.v K9, (t0)
138+
addi t0, t0, 16
139+
vle32.v K10, (t0)
140+
addi t0, t0, 16
141+
vle32.v K11, (t0)
142+
addi t0, t0, 16
143+
vle32.v K12, (t0)
144+
addi t0, t0, 16
145+
vle32.v K13, (t0)
146+
addi t0, t0, 16
147+
vle32.v K14, (t0)
148+
addi t0, t0, 16
149+
vle32.v K15, (t0)
150+
151+
// Setup mask for the vmerge to replace the first word (idx==0) in
152+
// message scheduling. There are 4 words, so an 8-bit mask suffices.
153+
vsetivli zero, 1, e8, m1, ta, ma
154+
vmv.v.i MASK, 0x01
155+
156+
// Load the state. The state is stored as {a,b,c,d,e,f,g,h}, but we
157+
// need {f,e,b,a},{h,g,d,c}. The dst vtype is e32m1 and the index vtype
158+
// is e8mf4. We use index-load with the i8 indices {20, 16, 4, 0},
159+
// loaded using the 32-bit little endian value 0x00041014.
160+
li t0, 0x00041014
161+
vsetivli zero, 1, e32, m1, ta, ma
162+
vmv.v.x INDICES, t0
163+
addi STATEP_C, STATEP, 8
164+
vsetivli zero, 4, e32, m1, ta, ma
165+
vluxei8.v FEBA, (STATEP), INDICES
166+
vluxei8.v HGDC, (STATEP_C), INDICES
167+
168+
.Lnext_block:
169+
addi NUM_BLOCKS, NUM_BLOCKS, -1
170+
171+
// Save the previous state, as it's needed later.
172+
vmv.v.v PREV_FEBA, FEBA
173+
vmv.v.v PREV_HGDC, HGDC
174+
175+
// Load the next 512-bit message block and endian-swap each 32-bit word.
176+
vle32.v W0, (DATA)
177+
vrev8.v W0, W0
178+
addi DATA, DATA, 16
179+
vle32.v W1, (DATA)
180+
vrev8.v W1, W1
181+
addi DATA, DATA, 16
182+
vle32.v W2, (DATA)
183+
vrev8.v W2, W2
184+
addi DATA, DATA, 16
185+
vle32.v W3, (DATA)
186+
vrev8.v W3, W3
187+
addi DATA, DATA, 16
188+
189+
// Do the 64 rounds of SHA-256.
190+
sha256_16rounds 0, K0, K1, K2, K3
191+
sha256_16rounds 0, K4, K5, K6, K7
192+
sha256_16rounds 0, K8, K9, K10, K11
193+
sha256_16rounds 1, K12, K13, K14, K15
194+
195+
// Add the previous state.
196+
vadd.vv FEBA, FEBA, PREV_FEBA
197+
vadd.vv HGDC, HGDC, PREV_HGDC
198+
199+
// Repeat if more blocks remain.
200+
bnez NUM_BLOCKS, .Lnext_block
201+
202+
// Store the new state and return.
203+
vsuxei8.v FEBA, (STATEP), INDICES
204+
vsuxei8.v HGDC, (STATEP_C), INDICES
205+
ret
206+
SYM_FUNC_END(sha256_transform_zvknha_or_zvknhb_zvkb)
207+
208+
.section ".rodata"
209+
.p2align 2
210+
.type K256, @object
211+
K256:
212+
.word 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5
213+
.word 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5
214+
.word 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3
215+
.word 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174
216+
.word 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc
217+
.word 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da
218+
.word 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7
219+
.word 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967
220+
.word 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13
221+
.word 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85
222+
.word 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3
223+
.word 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070
224+
.word 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5
225+
.word 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3
226+
.word 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208
227+
.word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
228+
.size K256, . - K256
229+

‎core/arch/riscv/crypto/sha256_riscv_zvkng_glue.c‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// SPDX-License-Identifier: BSD-2-Clause
2+
/*
3+
* Copyright (c) 2026, RISCstar Solutions Corporation
4+
*/
5+
6+
#include <crypto/crypto_accel.h>
7+
#include <kernel/thread.h>
8+
9+
void sha256_transform_zvknha_or_zvknhb_zvkb(uint32_t state[8],
10+
const void *src,
11+
unsigned int block_count);
12+
13+
void crypto_accel_sha256_compress(uint32_t state[8], const void *src,
14+
unsigned int block_count)
15+
{
16+
uint32_t vfp_state = thread_kernel_enable_vfp();
17+
18+
sha256_transform_zvknha_or_zvknhb_zvkb(state, src, block_count);
19+
thread_kernel_disable_vfp(vfp_state);
20+
}

0 commit comments

Comments
 (0)