aes/ni/
aes128.rs

Help
1use super::{arch::*, utils::*};
2use crate::{Block, Block8};
3use cipher::inout::InOut;
4use core::mem;
5
6/// AES-128 round keys
7pub(super) type RoundKeys = [__m128i; 11];
8
9#[inline]
10#[target_feature(enable = "aes")]
11pub(super) unsafe fn encrypt1(keys: &RoundKeys, block: InOut<'_, '_, Block>) {
12    let (in_ptr, out_ptr) = block.into_raw();
13    let mut b = _mm_loadu_si128(in_ptr as *const __m128i);
14    b = _mm_xor_si128(b, keys[0]);
15    b = _mm_aesenc_si128(b, keys[1]);
16    b = _mm_aesenc_si128(b, keys[2]);
17    b = _mm_aesenc_si128(b, keys[3]);
18    b = _mm_aesenc_si128(b, keys[4]);
19    b = _mm_aesenc_si128(b, keys[5]);
20    b = _mm_aesenc_si128(b, keys[6]);
21    b = _mm_aesenc_si128(b, keys[7]);
22    b = _mm_aesenc_si128(b, keys[8]);
23    b = _mm_aesenc_si128(b, keys[9]);
24    b = _mm_aesenclast_si128(b, keys[10]);
25    _mm_storeu_si128(out_ptr as *mut __m128i, b);
26}
27
28#[inline]
29#[target_feature(enable = "aes")]
30pub(super) unsafe fn encrypt8(keys: &RoundKeys, blocks: InOut<'_, '_, Block8>) {
31    let (in_ptr, out_ptr) = blocks.into_raw();
32    let mut b = load8(in_ptr);
33    xor8(&mut b, keys[0]);
34    aesenc8(&mut b, keys[1]);
35    aesenc8(&mut b, keys[2]);
36    aesenc8(&mut b, keys[3]);
37    aesenc8(&mut b, keys[4]);
38    aesenc8(&mut b, keys[5]);
39    aesenc8(&mut b, keys[6]);
40    aesenc8(&mut b, keys[7]);
41    aesenc8(&mut b, keys[8]);
42    aesenc8(&mut b, keys[9]);
43    aesenclast8(&mut b, keys[10]);
44    store8(out_ptr, b);
45}
46
47#[inline]
48#[target_feature(enable = "aes")]
49pub(super) unsafe fn decrypt1(keys: &RoundKeys, block: InOut<'_, '_, Block>) {
50    let (in_ptr, out_ptr) = block.into_raw();
51    let mut b = _mm_loadu_si128(in_ptr as *const __m128i);
52    b = _mm_xor_si128(b, keys[10]);
53    b = _mm_aesdec_si128(b, keys[9]);
54    b = _mm_aesdec_si128(b, keys[8]);
55    b = _mm_aesdec_si128(b, keys[7]);
56    b = _mm_aesdec_si128(b, keys[6]);
57    b = _mm_aesdec_si128(b, keys[5]);
58    b = _mm_aesdec_si128(b, keys[4]);
59    b = _mm_aesdec_si128(b, keys[3]);
60    b = _mm_aesdec_si128(b, keys[2]);
61    b = _mm_aesdec_si128(b, keys[1]);
62    b = _mm_aesdeclast_si128(b, keys[0]);
63    _mm_storeu_si128(out_ptr as *mut __m128i, b);
64}
65
66#[inline]
67#[target_feature(enable = "aes")]
68pub(super) unsafe fn decrypt8(keys: &RoundKeys, blocks: InOut<'_, '_, Block8>) {
69    let (in_ptr, out_ptr) = blocks.into_raw();
70    let mut b = load8(in_ptr);
71    xor8(&mut b, keys[10]);
72    aesdec8(&mut b, keys[9]);
73    aesdec8(&mut b, keys[8]);
74    aesdec8(&mut b, keys[7]);
75    aesdec8(&mut b, keys[6]);
76    aesdec8(&mut b, keys[5]);
77    aesdec8(&mut b, keys[4]);
78    aesdec8(&mut b, keys[3]);
79    aesdec8(&mut b, keys[2]);
80    aesdec8(&mut b, keys[1]);
81    aesdeclast8(&mut b, keys[0]);
82    store8(out_ptr, b);
83}
84
85macro_rules! expand_round {
86    ($keys:expr, $pos:expr, $round:expr) => {
87        let mut t1 = $keys[$pos - 1];
88        let mut t2;
89        let mut t3;
90
91        t2 = _mm_aeskeygenassist_si128(t1, $round);
92        t2 = _mm_shuffle_epi32(t2, 0xff);
93        t3 = _mm_slli_si128(t1, 0x4);
94        t1 = _mm_xor_si128(t1, t3);
95        t3 = _mm_slli_si128(t3, 0x4);
96        t1 = _mm_xor_si128(t1, t3);
97        t3 = _mm_slli_si128(t3, 0x4);
98        t1 = _mm_xor_si128(t1, t3);
99        t1 = _mm_xor_si128(t1, t2);
100
101        $keys[$pos] = t1;
102    };
103}
104
105#[inline]
106#[target_feature(enable = "aes")]
107pub(super) unsafe fn expand_key(key: &[u8; 16]) -> RoundKeys {
108    // SAFETY: `RoundKeys` is a `[__m128i; 11]` which can be initialized
109    // with all zeroes.
110    let mut keys: RoundKeys = mem::zeroed();
111
112    let k = _mm_loadu_si128(key.as_ptr() as *const __m128i);
113    keys[0] = k;
114
115    expand_round!(keys, 1, 0x01);
116    expand_round!(keys, 2, 0x02);
117    expand_round!(keys, 3, 0x04);
118    expand_round!(keys, 4, 0x08);
119    expand_round!(keys, 5, 0x10);
120    expand_round!(keys, 6, 0x20);
121    expand_round!(keys, 7, 0x40);
122    expand_round!(keys, 8, 0x80);
123    expand_round!(keys, 9, 0x1B);
124    expand_round!(keys, 10, 0x36);
125
126    keys
127}
128
129#[inline]
130#[target_feature(enable = "aes")]
131pub(super) unsafe fn inv_expanded_keys(keys: &RoundKeys) -> RoundKeys {
132    [
133        keys[0],
134        _mm_aesimc_si128(keys[1]),
135        _mm_aesimc_si128(keys[2]),
136        _mm_aesimc_si128(keys[3]),
137        _mm_aesimc_si128(keys[4]),
138        _mm_aesimc_si128(keys[5]),
139        _mm_aesimc_si128(keys[6]),
140        _mm_aesimc_si128(keys[7]),
141        _mm_aesimc_si128(keys[8]),
142        _mm_aesimc_si128(keys[9]),
143        keys[10],
144    ]
145}
aes/ni/aes128.rs

aes/ni/
aes128.rs
