CMSIS_5/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_s8.c

/*
 * Copyright (C) 2010-2020 Arm Limited or its affiliates. All rights reserved.
 *
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the License); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an AS IS BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* ----------------------------------------------------------------------
 * Project:      CMSIS NN Library
 * Title:        arm_fully_connected_s8
 * Description:  Fully connected function compatible with TF Lite.
 *
 * $Date:        April 1, 2020
 * $Revision:    V.1.5.0
 *
 * Target Processor:  Cortex-M and Cortex-A cores
 *
 * -------------------------------------------------------------------- */

#include "arm_math.h"
#include "arm_nnfunctions.h"

/**
 *  @ingroup groupNN
 */

/**
 * @addtogroup FC
 * @{
 */

/*
   * S8 basic fully-connected and matrix multiplication layer function for TensorFlow Lite
   *
   * Refer header file for details.
   *
   */

#if defined(ARM_MATH_MVEI)

arm_status
arm_fully_connected_s8(const int8_t *input,
                       const int8_t *kernel,
                       const uint16_t col_dim,
                       const uint16_t row_dim,
                       const uint16_t nb_batches,
                       const int32_t input_offset,
                       const int32_t filter_offset,
                       const int32_t out_mult,
                       const int32_t out_shift,
                       const int32_t output_offset,
                       const int32_t *bias,
                       int8_t *output,
                       const int32_t output_activation_min,
                       const int32_t output_activation_max,
                       q15_t *vec_buffer)
{
    (void)vec_buffer;
    const int8_t *input_a;
    const int32_t *bias_tmp = bias;
    const int8_t *weight_tmp = kernel;
    int32_t batch_count = nb_batches;

    const int16x8_t filter_offset_vec = vdupq_n_s16((int16_t)filter_offset);
    const int16x8_t input_offset_vec = vdupq_n_s16((int16_t)input_offset);

    while (batch_count)
    {
        bias_tmp = bias;
        weight_tmp = kernel;

        int cnt;
        cnt = row_dim >> 2;

        for (int out_c = 0; out_c < cnt; out_c++)
        {
            int32_t acc1 = *bias_tmp++;
            int32_t acc2 = *bias_tmp++;
            int32_t acc3 = *bias_tmp++;
            int32_t acc4 = *bias_tmp++;
            input_a = input;

            int16x8_t input_val, filter_val;
            int16x8_t tmp_a1, tmp_a2, tmp_a3, tmp_a4, tmp_b;
            int32x4_t acc;
            int32_t block_count;

            const int8_t *col = input_a;
            const int8_t *row_0 = weight_tmp;
            const int8_t *row_1 = weight_tmp + col_dim;
            const int8_t *row_2 = weight_tmp + 2 * col_dim;
            const int8_t *row_3 = weight_tmp + 3 * col_dim;

            block_count = col_dim >> 3U;

            while (block_count > 0U)
            {
                input_val = vldrbq_s16(col);
                tmp_b = vaddq_s16(input_val, input_offset_vec);

                filter_val = vldrbq_s16(row_0);
                tmp_a1 = vaddq_s16(filter_val, filter_offset_vec);
                acc1 = vmladavaq_s16(acc1, tmp_a1, tmp_b);

                filter_val = vldrbq_s16(row_1);
                tmp_a2 = vaddq_s16(filter_val, filter_offset_vec);
                acc2 = vmladavaq_s16(acc2, tmp_a2, tmp_b);

                filter_val = vldrbq_s16(row_2);
                tmp_a3 = vaddq_s16(filter_val, filter_offset_vec);
                acc3 = vmladavaq_s16(acc3, tmp_a3, tmp_b);

                filter_val = vldrbq_s16(row_3);
                tmp_a4 = vaddq_s16(filter_val, filter_offset_vec);
                acc4 = vmladavaq_s16(acc4, tmp_a4, tmp_b);

                col += 8;
                row_0 += 8;
                row_1 += 8;
                row_2 += 8;
                row_3 += 8;
                block_count--;
            }

            block_count = col_dim & 7;

            while (block_count > 0U)
            {
                q15_t col_ip = *col++;

                q7_t in_m1 = *row_0++;
                q7_t in_m2 = *row_1++;
                q7_t in_m3 = *row_2++;
                q7_t in_m4 = *row_3++;

                acc1 += (col_ip + input_offset) * (in_m1 + filter_offset);
                acc2 += (col_ip + input_offset) * (in_m2 + filter_offset);
                acc3 += (col_ip + input_offset) * (in_m3 + filter_offset);
                acc4 += (col_ip + input_offset) * (in_m4 + filter_offset);

                block_count--;
            }

            input_a = input + col_dim;
            weight_tmp += 4 * col_dim;

            acc[0] = acc1;
            acc[1] = acc2;
            acc[2] = acc3;
            acc[3] = acc4;

            acc = arm_requantize_mve(acc, out_mult, out_shift);
            acc = vaddq_s32(acc, vdupq_n_s32(output_offset));
            acc = vmaxq_s32(acc, vdupq_n_s32(output_activation_min));
            acc = vminq_s32(acc, vdupq_n_s32(output_activation_max));

            vstrbq_s32(output, acc);

            output += 4;
        }

        cnt = row_dim & 3;
        for (int out_c = 0; out_c < cnt; out_c++)
        {

            int32_t acc = *bias_tmp++;
            input_a = input;

            int16x8_t input_val, filter_val;
            int16x8_t tmp_a, tmp_b;
            int32_t block_count;

            const int8_t *col = input_a;
            const int8_t *kernel_cur = weight_tmp;

            block_count = col_dim >> 3U;

            while (block_count > 0U)
            {

                input_val = vldrbq_s16(col);
                filter_val = vldrbq_s16(kernel_cur);

                tmp_a = vaddq_s16(filter_val, filter_offset_vec);
                tmp_b = vaddq_s16(input_val, input_offset_vec);

                acc = vmladavaq_s16(acc, tmp_a, tmp_b);

                col += 8;
                kernel_cur += 8;
                block_count--;
            }

            block_count = col_dim & 7;

            while (block_count > 0U)
            {
                q15_t col_ip = *col++;
                q7_t in_m = *kernel_cur++;

                acc += (col_ip + input_offset) * (in_m + filter_offset);

                block_count--;
            }

            input_a += col_dim;
            weight_tmp += col_dim;

            acc = arm_nn_sat_doubling_high_mult(acc * (1 << LEFT_SHIFT(out_shift)), out_mult);
            acc = arm_nn_divide_by_power_of_two(acc, RIGHT_SHIFT(out_shift));

            acc += output_offset;

            acc = MAX(acc, output_activation_min);
            acc = MIN(acc, output_activation_max);

            *output++ = (int8_t)(acc);
        }
        input += col_dim;
        batch_count--;
    }
    return (ARM_MATH_SUCCESS);
}

#else
arm_status
arm_fully_connected_s8(const int8_t *input,
                       const int8_t *kernel,
                       const uint16_t col_dim,
                       const uint16_t row_dim,
                       const uint16_t nb_batches,
                       const int32_t input_offset,
                       const int32_t filter_offset,
                       const int32_t out_mult,
                       const int32_t out_shift,
                       const int32_t output_offset,
                       const int32_t *bias,
                       int8_t *output,
                       const int32_t output_activation_min,
                       const int32_t output_activation_max,
                       q15_t *vec_buffer)
{
    (void)vec_buffer;

    uint16_t batch_cnt = nb_batches;

    while (batch_cnt)
    {
        arm_nn_vec_mat_mult_t_s8(input,
                                 kernel,
                                 bias,
                                 output,
                                 input_offset,
                                 filter_offset,
                                 output_offset,
                                 out_mult,
                                 out_shift,
                                 col_dim,
                                 row_dim,
                                 output_activation_min,
                                 output_activation_max);
        input += col_dim;
        output += row_dim;
        batch_cnt--;
    }
    return (ARM_MATH_SUCCESS);
}
#endif /* ARM_MATH_HELIUM */

int32_t arm_fully_connected_s8_get_buffer_size(const uint16_t col_dim)
{
    (void)col_dim;
    return 0;
}

/**
 * @} end of FC group
 */