CMSIS_5/CMSIS/NN/Source/SoftmaxFunctions/arm_softmax_s8.c

/*
 * Copyright (C) 2010-2020 Arm Limited or its affiliates. All rights reserved.
 *
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the License); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an AS IS BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* ----------------------------------------------------------------------
 * Project:      CMSIS NN Library
 * Title:        arm_softmax_s8.c
 * Description:  S8 softmax function
 *
 * $Date:        01. March 2021
 * $Revision:    V.2.0.2
 *
 * Target Processor:  Cortex-M cores
 *
 * -------------------------------------------------------------------- */

#include "arm_nnfunctions.h"
#include "arm_nnsupportfunctions.h"

#define ACCUM_BITS 12

#ifdef ARM_MATH_MVEI
static int32x4_t arm_exp_on_negative_values_mve_32x4(int32x4_t val)
{
#define SHIFT_START (24)
    int32_t shift = SHIFT_START;
    int32x4_t mask;

    const int32x4_t val_mod_minus_quarter =
        vandq_s32(val, vdupq_n_s32((1 << SHIFT_START) - 1)) - vdupq_n_s32(1 << SHIFT_START);
    const int32x4_t remainder = vsubq_s32(val_mod_minus_quarter, val);
    const int32x4_t x = vaddq_n_s32(val_mod_minus_quarter << 5, 1 << 28);
    const int32x4_t x2 = MUL_SAT_MVE(x, x);
    const int32x4_t op_1 = DIV_POW2_MVE(MUL_SAT_MVE(x2, x2), 2) + MUL_SAT_MVE(x2, x);
    const int32x4_t op_2 = x + DIV_POW2_MVE(MUL_SAT_MVE(op_1, vdupq_n_s32(715827883)) + x2, 1);
    int32x4_t result = vdupq_n_s32(1895147668) + MUL_SAT_MVE(vdupq_n_s32(1895147668), op_2);

#define SELECT_IF_NON_ZERO(x)                                                                                          \
    {                                                                                                                  \
        mve_pred16_t p = vcmpneq_n_s32(remainder & vdupq_n_s32(1 << shift++), 0);                                      \
        mask = vmvnq_m_s32(vdupq_n_s32(0), vdupq_n_s32(0), p);                                                         \
        result = SELECT_USING_MASK(mask, MUL_SAT_MVE(result, vdupq_n_s32(x)), result);                                 \
    }

    SELECT_IF_NON_ZERO(1672461947)
    SELECT_IF_NON_ZERO(1302514674)
    SELECT_IF_NON_ZERO(790015084)
    SELECT_IF_NON_ZERO(290630308)
    SELECT_IF_NON_ZERO(39332535)
    SELECT_IF_NON_ZERO(720401)
    SELECT_IF_NON_ZERO(242)

#undef SELECT_IF_NON_ZERO

    mve_pred16_t p = vcmpeqq_n_s32(val, 0);
    mask = vmvnq_m_s32(vdupq_n_s32(0), vdupq_n_s32(0), p);

    result = SELECT_USING_MASK(mask, vdupq_n_s32(Q31_MAX), result);
    return result;
}
#endif

/**
 *  @ingroup groupNN
 */

/**
 * @addtogroup Softmax
 * @{
 */

void arm_softmax_s8(const int8_t *input,
                    const int32_t num_rows,
                    const int32_t row_size,
                    const int32_t mult,
                    const int32_t shift,
                    const int32_t diff_min,
                    int8_t *output)
{
#ifdef ARM_MATH_MVEI

#define ACT_MIN ((int8_t)Q7_MIN)
#define ACT_MAX ((int8_t)Q7_MAX)

    const int32_t mask = (1 << shift);

    for (int i_num_rows = 0; i_num_rows < num_rows; ++i_num_rows)
    {
        int8_t max = ACT_MIN;

        int32_t vec_count = (row_size + 15) / 16;
        uint32_t r_count = (uint32_t)row_size;
        for (int i = 0; i < vec_count; i++)
        {
            mve_pred16_t p = vctp8q(r_count);
            const int8x16_t ip = vldrbq_z_s8(&input[i * 16], p);
            max = vmaxvq_p_s8(max, ip, p);
            r_count -= 16;
        }

        vec_count = row_size / 4;
        int32_t idx = 0;
        int32_t sum = 0;

        while (vec_count)
        {
            int32x4_t ip = vldrbq_s32(&input[idx * 4]);
            ip = vsubq_n_s32(ip, max);
            mve_pred16_t p = vcmpgeq_n_s32(ip, diff_min);
            if (p != 0)
            {
                ip = vmulq_n_s32(ip, mask);

                int32x4_t res = MUL_SAT_MVE(ip, vdupq_n_s32(mult));

                res = arm_exp_on_negative_values_mve_32x4(res);
                res = DIV_POW2_MVE(res, ACCUM_BITS);
                res = vpselq_s32(res, vdupq_n_s32(0), p);
                sum += vaddvq_s32(res);
            }

            vec_count--;
            idx++;
        }

        const int32_t tail_idx = row_size & ~3;
        for (int i = 0; i < (row_size & 3); i++)
        {
            const int32_t diff = input[tail_idx + i] - max;
            if (diff >= diff_min)
            {
                sum += DIV_POW2(EXP_ON_NEG(MUL_SAT(diff * mask, mult)), ACCUM_BITS);
            }
        }

        const int32_t headroom = __CLZ((uint32_t)sum);
        const int32_t bits_over_unit = ACCUM_BITS - headroom + 23;
        const int32_t shifted_scale = ONE_OVER1((sum > 0 ? sum << headroom : 0) - (1 << 31));

        vec_count = row_size / 4;
        idx = 0;

        while (vec_count)
        {
            int32x4_t ip = vldrbq_s32(&input[idx]);
            ip = vsubq_n_s32(ip, max);

            mve_pred16_t p = vcmpgeq_n_s32(ip, diff_min);

            int32x4_t tmp_res;

            if (p != 0)
            {
                ip = vmulq_n_s32(ip, mask);

                tmp_res = MUL_SAT_MVE(ip, vdupq_n_s32(mult));
                tmp_res = arm_exp_on_negative_values_mve_32x4(tmp_res);
                tmp_res = MUL_SAT_MVE(vdupq_n_s32(shifted_scale), tmp_res);
                tmp_res = DIV_POW2_MVE(tmp_res, bits_over_unit);
                tmp_res += vdupq_n_s32(ACT_MIN);

                tmp_res = vmaxq_s32(tmp_res, vdupq_n_s32(ACT_MIN));
                tmp_res = vminq_s32(tmp_res, vdupq_n_s32(ACT_MAX));
                tmp_res = vpselq_s32(tmp_res, vdupq_n_s32(ACT_MIN), p);
            }
            else
            {
                tmp_res = vdupq_n_s32(ACT_MIN);
            }
            vstrbq_s32(&output[idx], tmp_res);
            vec_count--;
            idx += 4;
        }

        for (int i = 0; i < (row_size & 3); i++)
        {
            int32_t diff = input[tail_idx + i] - max;
            if (diff >= diff_min)
            {
                const int32_t res =
                    DIV_POW2(MUL_SAT(shifted_scale, EXP_ON_NEG(MUL_SAT(diff * mask, mult))), bits_over_unit) - 128;
                output[tail_idx + i] = (int8_t)CLAMP(res, (int32_t)ACT_MAX, (int32_t)ACT_MIN);
            }
            else
            {
                output[tail_idx + i] = ACT_MIN;
            }
        }

        input += row_size;
        output += row_size;
    }
#else
    const int32_t mask = (1 << shift);

    int32_t col = 0;
    int32_t row_idx;

    for (row_idx = 0; row_idx < num_rows; ++row_idx)
    {
        // Find the maximum value in order to ensure numerical stability
        int8_t max = *input;

        for (col = 1; col < row_size; ++col)
        {
            max = MAX(max, input[col]);
        }

        int32_t diff = 0;
        int32_t sum = 0;

        for (col = 0; col < row_size; ++col)
        {
            diff = input[col] - max;
            if (diff >= diff_min)
            {
                sum += DIV_POW2(EXP_ON_NEG(MUL_SAT(diff * mask, mult)), ACCUM_BITS);
            }
        }

        const int32_t headroom = __CLZ(sum);
        const int32_t bits_over_unit = ACCUM_BITS - headroom + 23;
        const int32_t shifted_scale = ONE_OVER1((sum > 0 ? sum << headroom : 0) - (1 << 31));

        for (col = 0; col < row_size; ++col)
        {
            diff = input[col] - max;
            if (diff >= diff_min)
            {
                const int32_t res =
                    DIV_POW2(MUL_SAT(shifted_scale, EXP_ON_NEG(MUL_SAT(diff * mask, mult))), bits_over_unit) - 128;
                output[col] = (int8_t)CLAMP(res, (int32_t)127, (int32_t)-128);
            }
            else
            {
                output[col] = -128;
            }
        }
        input += row_size;
        output += row_size;
    }

#endif
}
/**
 * @} end of Softmax group
 */