ring/aead/
aes_gcm.rs

Help
1// Copyright 2015-2016 Brian Smith.
2//
3// Permission to use, copy, modify, and/or distribute this software for any
4// purpose with or without fee is hereby granted, provided that the above
5// copyright notice and this permission notice appear in all copies.
6//
7// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES
8// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY
10// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
12// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
13// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
14
15use super::{
16    aes::{self, Counter},
17    block::{Block, BLOCK_LEN},
18    gcm, shift, Aad, Nonce, Tag,
19};
20use crate::{aead, cpu, error, polyfill::usize_from_u64_saturated};
21use core::ops::RangeFrom;
22
23/// AES-128 in GCM mode with 128-bit tags and 96 bit nonces.
24pub static AES_128_GCM: aead::Algorithm = aead::Algorithm {
25    key_len: 16,
26    init: init_128,
27    seal: aes_gcm_seal,
28    open: aes_gcm_open,
29    id: aead::AlgorithmID::AES_128_GCM,
30};
31
32/// AES-256 in GCM mode with 128-bit tags and 96 bit nonces.
33pub static AES_256_GCM: aead::Algorithm = aead::Algorithm {
34    key_len: 32,
35    init: init_256,
36    seal: aes_gcm_seal,
37    open: aes_gcm_open,
38    id: aead::AlgorithmID::AES_256_GCM,
39};
40
41#[derive(Clone)]
42pub struct Key {
43    gcm_key: gcm::Key, // First because it has a large alignment requirement.
44    aes_key: aes::Key,
45}
46
47fn init_128(key: &[u8], cpu_features: cpu::Features) -> Result<aead::KeyInner, error::Unspecified> {
48    init(key, aes::Variant::AES_128, cpu_features)
49}
50
51fn init_256(key: &[u8], cpu_features: cpu::Features) -> Result<aead::KeyInner, error::Unspecified> {
52    init(key, aes::Variant::AES_256, cpu_features)
53}
54
55fn init(
56    key: &[u8],
57    variant: aes::Variant,
58    cpu_features: cpu::Features,
59) -> Result<aead::KeyInner, error::Unspecified> {
60    let aes_key = aes::Key::new(key, variant, cpu_features)?;
61    let gcm_key = gcm::Key::new(
62        aes_key.encrypt_block(Block::zero(), cpu_features),
63        cpu_features,
64    );
65    Ok(aead::KeyInner::AesGcm(Key { gcm_key, aes_key }))
66}
67
68const CHUNK_BLOCKS: usize = 3 * 1024 / 16;
69
70fn aes_gcm_seal(
71    key: &aead::KeyInner,
72    nonce: Nonce,
73    aad: Aad<&[u8]>,
74    in_out: &mut [u8],
75    cpu_features: cpu::Features,
76) -> Result<Tag, error::Unspecified> {
77    let Key { gcm_key, aes_key } = match key {
78        aead::KeyInner::AesGcm(key) => key,
79        _ => unreachable!(),
80    };
81
82    let mut auth = gcm::Context::new(gcm_key, aad, in_out.len(), cpu_features)?;
83
84    let mut ctr = Counter::one(nonce);
85    let tag_iv = ctr.increment();
86
87    #[cfg(target_arch = "x86_64")]
88    let in_out = {
89        if !aes_key.is_aes_hw(cpu_features) || !auth.is_avx() {
90            in_out
91        } else {
92            use crate::c;
93            let (htable, xi) = auth.inner();
94            prefixed_extern! {
95                // `HTable` and `Xi` should be 128-bit aligned. TODO: Can we shrink `HTable`? The
96                // assembly says it needs just nine values in that array.
97                fn aesni_gcm_encrypt(
98                    input: *const u8,
99                    output: *mut u8,
100                    len: c::size_t,
101                    key: &aes::AES_KEY,
102                    ivec: &mut Counter,
103                    Htable: &gcm::HTable,
104                    Xi: &mut gcm::Xi) -> c::size_t;
105            }
106            let processed = unsafe {
107                aesni_gcm_encrypt(
108                    in_out.as_ptr(),
109                    in_out.as_mut_ptr(),
110                    in_out.len(),
111                    aes_key.inner_less_safe(),
112                    &mut ctr,
113                    htable,
114                    xi,
115                )
116            };
117
118            &mut in_out[processed..]
119        }
120    };
121
122    #[cfg(target_arch = "aarch64")]
123    let in_out = {
124        if !aes_key.is_aes_hw(cpu_features) || !auth.is_clmul() {
125            in_out
126        } else {
127            let whole_block_bits = auth.in_out_whole_block_bits();
128            if whole_block_bits.as_bits() > 0 {
129                use crate::{bits::BitLength, c};
130                let (htable, xi) = auth.inner();
131                prefixed_extern! {
132                    fn aes_gcm_enc_kernel(
133                        input: *const u8,
134                        in_bits: BitLength<c::size_t>,
135                        output: *mut u8,
136                        Xi: &mut gcm::Xi,
137                        ivec: &mut Counter,
138                        key: &aes::AES_KEY,
139                        Htable: &gcm::HTable);
140                }
141                unsafe {
142                    aes_gcm_enc_kernel(
143                        in_out.as_ptr(),
144                        whole_block_bits,
145                        in_out.as_mut_ptr(),
146                        xi,
147                        &mut ctr,
148                        aes_key.inner_less_safe(),
149                        htable,
150                    )
151                }
152            }
153
154            &mut in_out[whole_block_bits.as_usize_bytes_rounded_up()..]
155        }
156    };
157
158    let (whole, remainder) = {
159        let in_out_len = in_out.len();
160        let whole_len = in_out_len - (in_out_len % BLOCK_LEN);
161        in_out.split_at_mut(whole_len)
162    };
163
164    for chunk in whole.chunks_mut(CHUNK_BLOCKS * BLOCK_LEN) {
165        aes_key.ctr32_encrypt_within(chunk, 0.., &mut ctr, cpu_features);
166        auth.update_blocks(chunk);
167    }
168
169    if !remainder.is_empty() {
170        let mut input = Block::zero();
171        input.overwrite_part_at(0, remainder);
172        let mut output = aes_key.encrypt_iv_xor_block(ctr.into(), input, cpu_features);
173        output.zero_from(remainder.len());
174        auth.update_block(output);
175        remainder.copy_from_slice(&output.as_ref()[..remainder.len()]);
176    }
177
178    Ok(finish(aes_key, auth, tag_iv))
179}
180
181fn aes_gcm_open(
182    key: &aead::KeyInner,
183    nonce: Nonce,
184    aad: Aad<&[u8]>,
185    in_out: &mut [u8],
186    src: RangeFrom<usize>,
187    cpu_features: cpu::Features,
188) -> Result<Tag, error::Unspecified> {
189    let Key { gcm_key, aes_key } = match key {
190        aead::KeyInner::AesGcm(key) => key,
191        _ => unreachable!(),
192    };
193
194    let mut auth = {
195        let unprefixed_len = in_out
196            .len()
197            .checked_sub(src.start)
198            .ok_or(error::Unspecified)?;
199        gcm::Context::new(gcm_key, aad, unprefixed_len, cpu_features)
200    }?;
201
202    let mut ctr = Counter::one(nonce);
203    let tag_iv = ctr.increment();
204
205    let in_prefix_len = src.start;
206
207    #[cfg(target_arch = "x86_64")]
208    let in_out = {
209        if !aes_key.is_aes_hw(cpu_features) || !auth.is_avx() {
210            in_out
211        } else {
212            use crate::c;
213            let (htable, xi) = auth.inner();
214            prefixed_extern! {
215                // `HTable` and `Xi` should be 128-bit aligned. TODO: Can we shrink `HTable`? The
216                // assembly says it needs just nine values in that array.
217                fn aesni_gcm_decrypt(
218                    input: *const u8,
219                    output: *mut u8,
220                    len: c::size_t,
221                    key: &aes::AES_KEY,
222                    ivec: &mut Counter,
223                    Htable: &gcm::HTable,
224                    Xi: &mut gcm::Xi) -> c::size_t;
225            }
226
227            let processed = unsafe {
228                aesni_gcm_decrypt(
229                    in_out[src.clone()].as_ptr(),
230                    in_out.as_mut_ptr(),
231                    in_out.len() - src.start,
232                    aes_key.inner_less_safe(),
233                    &mut ctr,
234                    htable,
235                    xi,
236                )
237            };
238            &mut in_out[processed..]
239        }
240    };
241
242    #[cfg(target_arch = "aarch64")]
243    let in_out = {
244        if !aes_key.is_aes_hw(cpu_features) || !auth.is_clmul() {
245            in_out
246        } else {
247            let whole_block_bits = auth.in_out_whole_block_bits();
248            if whole_block_bits.as_bits() > 0 {
249                use crate::{bits::BitLength, c};
250                let (htable, xi) = auth.inner();
251                prefixed_extern! {
252                    fn aes_gcm_dec_kernel(
253                        input: *const u8,
254                        in_bits: BitLength<c::size_t>,
255                        output: *mut u8,
256                        Xi: &mut gcm::Xi,
257                        ivec: &mut Counter,
258                        key: &aes::AES_KEY,
259                        Htable: &gcm::HTable);
260                }
261
262                unsafe {
263                    aes_gcm_dec_kernel(
264                        in_out[src.clone()].as_ptr(),
265                        whole_block_bits,
266                        in_out.as_mut_ptr(),
267                        xi,
268                        &mut ctr,
269                        aes_key.inner_less_safe(),
270                        htable,
271                    )
272                }
273            }
274
275            &mut in_out[whole_block_bits.as_usize_bytes_rounded_up()..]
276        }
277    };
278
279    let whole_len = {
280        let in_out_len = in_out.len() - in_prefix_len;
281        in_out_len - (in_out_len % BLOCK_LEN)
282    };
283    {
284        let mut chunk_len = CHUNK_BLOCKS * BLOCK_LEN;
285        let mut output = 0;
286        let mut input = in_prefix_len;
287        loop {
288            if whole_len - output < chunk_len {
289                chunk_len = whole_len - output;
290            }
291            if chunk_len == 0 {
292                break;
293            }
294
295            auth.update_blocks(&in_out[input..][..chunk_len]);
296            aes_key.ctr32_encrypt_within(
297                &mut in_out[output..][..(chunk_len + in_prefix_len)],
298                in_prefix_len..,
299                &mut ctr,
300                cpu_features,
301            );
302            output += chunk_len;
303            input += chunk_len;
304        }
305    }
306
307    let remainder = &mut in_out[whole_len..];
308    shift::shift_partial((in_prefix_len, remainder), |remainder| {
309        let mut input = Block::zero();
310        input.overwrite_part_at(0, remainder);
311        auth.update_block(input);
312        aes_key.encrypt_iv_xor_block(ctr.into(), input, cpu_features)
313    });
314
315    Ok(finish(aes_key, auth, tag_iv))
316}
317
318fn finish(aes_key: &aes::Key, gcm_ctx: gcm::Context, tag_iv: aes::Iv) -> Tag {
319    // Finalize the tag and return it.
320    gcm_ctx.pre_finish(|pre_tag, cpu_features| {
321        let encrypted_iv = aes_key.encrypt_block(tag_iv.into_block_less_safe(), cpu_features);
322        let tag = pre_tag ^ encrypted_iv;
323        Tag(*tag.as_ref())
324    })
325}
326
327pub(super) const MAX_IN_OUT_LEN: usize = super::max_input_len(BLOCK_LEN, 2);
328
329// [NIST SP800-38D] Section 5.2.1.1. Note that [RFC 5116 Section 5.1] and
330// [RFC 5116 Section 5.2] have an off-by-one error in `P_MAX`.
331//
332// [NIST SP800-38D]:
333//    http://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38d.pdf
334// [RFC 5116 Section 5.1]: https://tools.ietf.org/html/rfc5116#section-5.1
335// [RFC 5116 Section 5.2]: https://tools.ietf.org/html/rfc5116#section-5.2
336const _MAX_INPUT_LEN_BOUNDED_BY_NIST: () =
337    assert!(MAX_IN_OUT_LEN == usize_from_u64_saturated(((1u64 << 39) - 256) / 8));
ring/aead/aes_gcm.rs

ring/aead/
aes_gcm.rs
