diff --git a/scripts/automate_skiplist.sh b/scripts/automate_skiplist.sh new file mode 100755 index 0000000000..ad5f52f253 --- /dev/null +++ b/scripts/automate_skiplist.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +# The target platform +PLATFORM=$1 + +if [ "$#" -ne 1 ]; then + echo "Please provide the platform name. Usage: $0 arg" + exit 1 +fi + +BASE=$(cd $(dirname "$0")/../.. && pwd) +TRITON_PROJ=${BASE}/intel-xpu-backend-for-triton + +# Run core test, regression test and interpreter test in mode unskip +. ${TRITON_PROJ}/scripts/test-triton.sh --core --interpreter --unskip --reports + +# Parse logs and get all failed cases for all categories +TXT_DIR=${TRITON_PROJ}/scripts/skiplist/${PLATFORM} +mkdir -p "${TXT_DIR}" + +for xml_file in ${TRITON_TEST_REPORTS_DIR}/*.xml; do + file_name=$(basename ${xml_file} .xml) + OUT_FILE=${TXT_DIR}/$file_name.txt + + python ${TRITON_PROJ}/scripts/get_failed_cases.py ${xml_file} ${OUT_FILE} +done diff --git a/scripts/get_failed_cases.py b/scripts/get_failed_cases.py new file mode 100644 index 0000000000..57cb9e9d3a --- /dev/null +++ b/scripts/get_failed_cases.py @@ -0,0 +1,41 @@ +import xml.etree.ElementTree as ET +import argparse + + +def create_argument_parser() -> argparse.ArgumentParser: + """Create argument parser.""" + parser = argparse.ArgumentParser() + parser.add_argument('input_file', type=str, help='input XML file') + parser.add_argument('output_file', type=str, help='output TXT file') + return parser + + +def extract_failed_from_xml(in_file: str, out_file: str): + """Process XML log file and output failed cases.""" + root = ET.parse(in_file).getroot() + failed = [] + + for testcase in root.findall('.//testcase'): + for child in testcase: + if child.tag == 'error' or child.tag == 'failure': + classname = testcase.get('classname').replace('.', '/') + '.py' + case = testcase.get('name') + result = f'{classname}::{case}' + failed.append(result) + + if len(failed) == 0: + return + + with open(out_file, 'w') as f: + for result in failed: + f.write(result + '\n') + + +def main(): + """Main.""" + args = create_argument_parser().parse_args() + extract_failed_from_xml(args.input_file, args.output_file) + + +if __name__ == '__main__': + main() diff --git a/scripts/pytest-utils.sh b/scripts/pytest-utils.sh index da2cdcdb3c..726f2f53dd 100644 --- a/scripts/pytest-utils.sh +++ b/scripts/pytest-utils.sh @@ -7,6 +7,9 @@ TRITON_TEST_SKIPLIST_DIR="${TRITON_TEST_SKIPLIST_DIR:-$SCRIPTS_DIR/skiplist/defa TRITON_TEST_WARNING_REPORTS="${TRITON_TEST_WARNING_REPORTS:-false}" TRITON_TEST_IGNORE_ERRORS="${TRITON_TEST_IGNORE_ERRORS:-false}" +if [[ $TEST_UNSKIP = true ]]; then + TRITON_TEST_IGNORE_ERRORS=true +fi # absolute path for the selected skip list TRITON_TEST_SKIPLIST_DIR="$(cd "$TRITON_TEST_SKIPLIST_DIR" && pwd)" # absolute path for the current skip list diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh index d757d81c4a..f0a52d90ef 100755 --- a/scripts/test-triton.sh +++ b/scripts/test-triton.sh @@ -7,6 +7,7 @@ export PIP_DISABLE_PIP_VERSION_CHECK=1 # Select which tests to run. TEST_MICRO_BENCHMARKS=false TEST_CORE=false +TEST_INTERPRETER=false TEST_TUTORIAL=false TEST_UNIT=false VENV=false @@ -30,6 +31,10 @@ for arg in "$@"; do TEST_CORE=true shift ;; + --interpreter) + TEST_INTERPRETER=true + shift + ;; --tutorial) TEST_TUTORIAL=true shift @@ -69,7 +74,8 @@ for arg in "$@"; do esac done -if [ "$TEST_MICRO_BENCHMARKS" = false ] && [ "$TEST_CORE" = false ] && [ "$TEST_TUTORIAL" = false ] && [ "$TEST_UNIT" = false ]; then +# Only run interpreter test when $TEST_INTERPRETER is ture +if [ "$TEST_MICRO_BENCHMARKS" = false ] && [ "$TEST_CORE" = false ] && [ "$TEST_INTERPRETER" = false ] && [ "$TEST_TUTORIAL" = false ] && [ "$TEST_UNIT" = false ]; then TEST_MICRO_BENCHMARKS=true TEST_CORE=true TEST_TUTORIAL=true @@ -186,6 +192,22 @@ run_regression_tests() { pytest -vvv -s --device xpu . --reruns 10 --ignore=test_performance.py } +run_interpreter_tests() { + echo "***************************************************" + echo "****** Running Triton Interpreter tests ******" + echo "***************************************************" + INTERPRETER_TEST_DIR=$TRITON_PROJ/python/test/unit + + if [ ! -d "${INTERPRETER_TEST_DIR}" ]; then + echo "Not found '${INTERPRETER_TEST_DIR}'. Build Triton please" ; exit 3 + fi + cd ${INTERPRETER_TEST_DIR} + export TEST_UNSKIP + TRITON_INTERPRET=1 TRITON_TEST_SUITE=interpreter \ + pytest -vvv -n 16 -m interpreter language/test_core.py language/test_standard.py \ + language/test_random.py --device cpu +} + run_tutorial_tests() { echo "***************************************************" echo "**** Running Triton Tutorial tests ******" @@ -218,6 +240,9 @@ test_triton() { run_core_tests run_regression_tests fi + if [ "$TEST_INTERPRETER" = true ]; then + run_interpreter_tests + fi if [ "$TEST_TUTORIAL" = true ]; then run_tutorial_tests fi