wasm-micro-runtime/tests/wamr-test-suites/requirement-engineering-test-script/run_requirement.py

#!/usr/bin/python3
#
# Copyright (C) 2019 Intel Corporation.  All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#

import os
import sys
import argparse
from typing import Dict, Tuple, List
import importlib
import inspect
import csv

REQUIREMENT_TESTS_DIR = "../../requirement-engineering"
SUBREQUIREMENT_DESCRIPTIONS = {}


# To use this empty function to do signature check
def expected_build_func_template(verbose: bool) -> None:
    pass


# To use this empty function to do signature check
# The actual implementation of the return value should has following information:
#
def expected_run_func_template(
    output_dir: str, subrequirement_ids: List[int]
) -> Dict[int, Dict[Tuple[str, str], bool]]:
    pass


def dynamic_import(requirement_dir: str):
    # Declare that we intend to modify the global variable
    global SUBREQUIREMENT_DESCRIPTIONS
    sys.path.append(requirement_dir)
    os.chdir(requirement_dir)

    try:
        build_module = importlib.import_module("build")
        build_function = getattr(build_module, "build")
    except AttributeError:
        raise ImportError("'build' function not found in the specified build.py file.")

    try:
        run_module = importlib.import_module("run")
        run_function = getattr(run_module, "run")
        SUBREQUIREMENT_DESCRIPTIONS = getattr(run_module, "SUBREQUIREMENT_DESCRIPTIONS")
    except AttributeError:
        raise ImportError(
            "'run' function or 'SUBREQUIREMENT_DESCRIPTIONS' not found in the specified run.py file."
        )

    # Do signature check
    expected_signature = inspect.signature(expected_build_func_template)
    actual_signature = inspect.signature(build_function)
    assert (
        actual_signature == expected_signature
    ), "The build function doesn't have the expected signature"

    expected_signature = inspect.signature(expected_run_func_template)
    actual_signature = inspect.signature(run_function)
    assert (
        actual_signature == expected_signature
    ), "The run function doesn't have the expected signature"

    # Check if the variable is a dictionary
    if not isinstance(SUBREQUIREMENT_DESCRIPTIONS, dict):
        raise TypeError("SUBREQUIREMENT_DESCRIPTIONS is not a dictionary")

    # Check the types of keys and values in the dictionary
    for key, value in SUBREQUIREMENT_DESCRIPTIONS.items():
        if not isinstance(key, int):
            raise TypeError("Key in SUBREQUIREMENT_DESCRIPTIONS is not an int")
        if not (
            isinstance(value, tuple)
            and len(value) == 2
            and all(isinstance(elem, str) for elem in value)
        ):
            raise TypeError(
                "Value in SUBREQUIREMENT_DESCRIPTIONS is not a Tuple[str, str]"
            )

    return build_function, run_function


def cmd_line_summary(
    requirement_name: str, result_dict: dict, subrequirement_descriptions: dict
):
    # command line summary
    total, total_pass_nums, total_fail_nums = 0, 0, 0
    print(f"\n============ Start: Summary of {requirement_name} test ============")
    for subrequirement_id in result_dict.keys():
        sub_total = len(result_dict[subrequirement_id])
        pass_nums = len(
            [_ for _, result in result_dict[subrequirement_id].items() if result]
        )
        fail_nums = len(
            [_ for _, result in result_dict[subrequirement_id].items() if not result]
        )
        issue_number, subrequirement_description = subrequirement_descriptions.get(
            subrequirement_id, ""
        )

        print(f"\nTest Sub-requirement id: {subrequirement_id}")
        print(f"Issue Number: {issue_number}")
        print(f"Sub-requirement description: {subrequirement_description}")
        print(f"Number of test cases: {sub_total}")
        print(f"Pass: {pass_nums}")
        print(f"Fail: {fail_nums}\n")
        print(
            "----------------------------------------------------------------------------"
        )
        total += sub_total
        total_pass_nums += pass_nums
        total_fail_nums += fail_nums

    print(f"\nTotal Number of test cases: {total}")
    print(f"Pass: {total_pass_nums}")
    print(f"Fail: {total_fail_nums}\n")

    print(f"============= End: Summary of {requirement_name} test =============\n")


def generate_report(output_filename: str, result_dict: dict):
    # create a list of column names
    column_names = [
        "subrequirement id",
        "issue number",
        "subrequirement description",
        "running mode",
        "test case name",
        "test case description",
        "test case executing result",
    ]

    # open the output file in write mode
    with open(output_filename + ".csv", "w") as output_file:
        # create a csv writer object
        csv_writer = csv.writer(output_file)
        # write the column names as the first row
        csv_writer.writerow(column_names)
        # loop through the result_dict
        for subrequirement_id, test_cases in result_dict.items():
            # get the subrequirement description from the subrequirement_descriptions dict
            issue_number, subrequirement_description = SUBREQUIREMENT_DESCRIPTIONS.get(
                subrequirement_id, ""
            )
            # loop through the test cases
            for test_case, result in test_cases.items():
                # unpack the test case name and description from the tuple
                test_case_name, test_case_description = test_case
                # convert the result to pass or fail
                result = "pass" if result else "fail"
                # create a list of values for the current row
                row_values = [
                    subrequirement_id,
                    issue_number,
                    subrequirement_description,
                    "AOT",
                    test_case_name,
                    test_case_description,
                    result,
                ]
                # write the row values to the output file
                csv_writer.writerow(row_values)


def run_requirement(
    requirement_name: str, output_dir: str, subrequirement_ids: List[int]
):
    requirement_dir = os.path.join(REQUIREMENT_TESTS_DIR, requirement_name)
    if not os.path.isdir(requirement_dir):
        print(f"No such requirement in directory {requirement_dir} exists")
        sys.exit(1)

    output_path = os.path.join(output_dir, requirement_name)

    build_requirement_func, run_requirement_func = dynamic_import(requirement_dir)

    build_requirement_func(verbose=False)
    result_dict = run_requirement_func(output_path, subrequirement_ids)

    cmd_line_summary(requirement_name, result_dict, SUBREQUIREMENT_DESCRIPTIONS)
    generate_report(output_path, result_dict)


def main():
    parser = argparse.ArgumentParser(description="Process command line options.")

    # Define the '-o' option for output directory
    parser.add_argument(
        "-o", "--output_directory", required=True, help="Report output directory"
    )

    # Define the '-r' option for requirement name
    parser.add_argument(
        "-r", "--requirement_name", required=True, help="Requirement name"
    )

    # Define the subrequirement IDs as a list of integers
    parser.add_argument(
        "subrequirement_ids", nargs="*", type=int, help="Subrequirement IDs (optional)"
    )

    # Parse the arguments
    args = parser.parse_args()

    run_requirement(
        args.requirement_name, args.output_directory, list(args.subrequirement_ids)
    )


if __name__ == "__main__":
    main()