[UT] Automatically generate skiplist when on a new platform (#1596)

# Functionality

This PR fix
#1423

# Use

`./scripts/automate_skiplist.sh  ${NEW_PLATFORM}`  gives you 
`./scripts/skiplist/${NEW_PLATFORM}/....txt`

scripts/automate_skiplist.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# The target platform
+PLATFORM=$1
+
+if [ "$#" -ne 1 ]; then
+   echo "Please provide the platform name. Usage: $0 arg"
+   exit 1
+fi
+
+BASE=$(cd $(dirname "$0")/../.. && pwd)
+TRITON_PROJ=${BASE}/intel-xpu-backend-for-triton
+
+# Run core test, regression test and interpreter test in mode unskip
+. ${TRITON_PROJ}/scripts/test-triton.sh --core --interpreter --unskip --reports
+
+# Parse logs and get all failed cases for all categories
+TXT_DIR=${TRITON_PROJ}/scripts/skiplist/${PLATFORM}
+mkdir -p "${TXT_DIR}"
+
+for xml_file in ${TRITON_TEST_REPORTS_DIR}/*.xml; do
+    file_name=$(basename ${xml_file} .xml)
+    OUT_FILE=${TXT_DIR}/$file_name.txt
+
+    python ${TRITON_PROJ}/scripts/get_failed_cases.py ${xml_file} ${OUT_FILE}
+done

scripts/get_failed_cases.py

@@ -0,0 +1,41 @@
+import xml.etree.ElementTree as ET
+import argparse
+
+
+def create_argument_parser() -> argparse.ArgumentParser:
+    """Create argument parser."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument('input_file', type=str, help='input XML file')
+    parser.add_argument('output_file', type=str, help='output TXT file')
+    return parser
+
+
+def extract_failed_from_xml(in_file: str, out_file: str):
+    """Process XML log file and output failed cases."""
+    root = ET.parse(in_file).getroot()
+    failed = []
+
+    for testcase in root.findall('.//testcase'):
+        for child in testcase:
+            if child.tag == 'error' or child.tag == 'failure':
+                classname = testcase.get('classname').replace('.', '/') + '.py'
+                case = testcase.get('name')
+                result = f'{classname}::{case}'
+                failed.append(result)
+
+    if len(failed) == 0:
+        return
+
+    with open(out_file, 'w') as f:
+        for result in failed:
+            f.write(result + '\n')
+
+
+def main():
+    """Main."""
+    args = create_argument_parser().parse_args()
+    extract_failed_from_xml(args.input_file, args.output_file)
+
+
+if __name__ == '__main__':
+    main()

scripts/pytest-utils.sh

@@ -7,6 +7,9 @@ TRITON_TEST_SKIPLIST_DIR="${TRITON_TEST_SKIPLIST_DIR:-$SCRIPTS_DIR/skiplist/defa
 TRITON_TEST_WARNING_REPORTS="${TRITON_TEST_WARNING_REPORTS:-false}"
 TRITON_TEST_IGNORE_ERRORS="${TRITON_TEST_IGNORE_ERRORS:-false}"
 
+if [[ $TEST_UNSKIP = true ]]; then
+    TRITON_TEST_IGNORE_ERRORS=true
+fi
 # absolute path for the selected skip list
 TRITON_TEST_SKIPLIST_DIR="$(cd "$TRITON_TEST_SKIPLIST_DIR" && pwd)"
 # absolute path for the current skip list

scripts/test-triton.sh

@@ -7,6 +7,7 @@ export PIP_DISABLE_PIP_VERSION_CHECK=1
 # Select which tests to run.
 TEST_MICRO_BENCHMARKS=false
 TEST_CORE=false
+TEST_INTERPRETER=false
 TEST_TUTORIAL=false
 TEST_UNIT=false
 VENV=false
@@ -30,6 +31,10 @@ for arg in "$@"; do
       TEST_CORE=true
       shift
       ;;
+    --interpreter)
+      TEST_INTERPRETER=true
+      shift
+      ;;
     --tutorial)
       TEST_TUTORIAL=true
       shift
@@ -69,7 +74,8 @@ for arg in "$@"; do
   esac
 done
 
-if [ "$TEST_MICRO_BENCHMARKS" = false ] && [ "$TEST_CORE" = false ] && [ "$TEST_TUTORIAL" = false ] && [ "$TEST_UNIT" = false ]; then
+# Only run interpreter test when $TEST_INTERPRETER is ture
+if [ "$TEST_MICRO_BENCHMARKS" = false ] && [ "$TEST_CORE" = false ] && [ "$TEST_INTERPRETER" = false ] && [ "$TEST_TUTORIAL" = false ] && [ "$TEST_UNIT" = false ]; then
   TEST_MICRO_BENCHMARKS=true
   TEST_CORE=true
   TEST_TUTORIAL=true
@@ -186,6 +192,22 @@ run_regression_tests() {
   pytest -vvv -s --device xpu . --reruns 10 --ignore=test_performance.py
 }
 
+run_interpreter_tests() {
+  echo "***************************************************"
+  echo "******   Running Triton Interpreter tests    ******"
+  echo "***************************************************"
+  INTERPRETER_TEST_DIR=$TRITON_PROJ/python/test/unit
+
+  if [ ! -d "${INTERPRETER_TEST_DIR}" ]; then
+    echo "Not found '${INTERPRETER_TEST_DIR}'. Build Triton please" ; exit 3
+  fi
+  cd ${INTERPRETER_TEST_DIR}
+  export TEST_UNSKIP
+  TRITON_INTERPRET=1 TRITON_TEST_SUITE=interpreter \
+  pytest -vvv -n 16 -m interpreter language/test_core.py language/test_standard.py \
+  language/test_random.py --device cpu
+}
+
 run_tutorial_tests() {
   echo "***************************************************"
   echo "**** Running Triton Tutorial tests           ******"
@@ -218,6 +240,9 @@ test_triton() {
     run_core_tests
     run_regression_tests
   fi
+  if [ "$TEST_INTERPRETER" = true ]; then
+    run_interpreter_tests
+  fi
   if [ "$TEST_TUTORIAL" = true ]; then
     run_tutorial_tests
   fi