From 404ec40a8d5ec194de9c154d2864e934b65c3d46 Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Tue, 19 Mar 2024 15:27:34 +0100 Subject: [PATCH 1/4] Add test for INC examples --- .github/workflows/test_inc_examples.yml | 47 +++++++++++++++++++++ examples/neural_compressor/test_examples.py | 22 +++------- 2 files changed, 53 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/test_inc_examples.yml diff --git a/.github/workflows/test_inc_examples.yml b/.github/workflows/test_inc_examples.yml new file mode 100644 index 0000000000..a5c735cb14 --- /dev/null +++ b/.github/workflows/test_inc_examples.yml @@ -0,0 +1,47 @@ +name: INC - Examples Test + +on: + workflow_dispatch: + schedule: + - cron: 0 1 * * 1 # run weekly: every Monday at 1am + push: + paths: + - '.github/workflows/test_inc_examples.yml' + - 'examples/neural_compressor/*' + pull_request: + paths: + - '.github/workflows/test_inc_examples.yml' + - 'examples/neural_compressor/*' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.10"] + + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + pip install optimum[neural-compressor,ipex] pytest + pip install -r examples/neural_compressor/text-classification/requirements.txt + pip install -r examples/neural_compressor/question-answering/requirements.txt + pip install -r examples/neural_compressor/token-classification/requirements.txt + pip install -r examples/neural_compressor/language-modeling/requirements.txt + pip install -r examples/neural_compressor/multiple-choice/requirements.txt + + - name: Test examples + run: | + python -m pytest examples/neural_compressor/test_examples.py \ No newline at end of file diff --git a/examples/neural_compressor/test_examples.py b/examples/neural_compressor/test_examples.py index 89779ffea3..b4be6064b2 100644 --- a/examples/neural_compressor/test_examples.py +++ b/examples/neural_compressor/test_examples.py @@ -59,7 +59,7 @@ def test_run_glue(self): with tempfile.TemporaryDirectory() as tmp_dir: test_args = f""" run_glue.py - --model_name_or_path distilbert-base-uncased-finetuned-sst-2-english + --model_name_or_path hf-internal-testing/tiny-random-DistilBertForSequenceClassification --task_name sst2 --apply_quantization --apply_pruning @@ -79,13 +79,12 @@ def test_run_glue(self): with patch.object(sys, "argv", test_args): run_glue.main() results = get_results(tmp_dir) - self.assertGreaterEqual(results["eval_accuracy"], 0.70) def test_run_qa(self): with tempfile.TemporaryDirectory() as tmp_dir: test_args = f""" run_qa.py - --model_name_or_path distilbert-base-uncased-distilled-squad + --model_name_or_path hf-internal-testing/tiny-random-DistilBertForQuestionAnswering --dataset_name squad --apply_quantization --apply_pruning @@ -105,14 +104,12 @@ def test_run_qa(self): with patch.object(sys, "argv", test_args): run_qa.main() results = get_results(tmp_dir) - self.assertGreaterEqual(results["eval_f1"], 70) - self.assertGreaterEqual(results["eval_exact_match"], 70) def test_run_ner(self): with tempfile.TemporaryDirectory() as tmp_dir: test_args = f""" run_ner.py - --model_name_or_path elastic/distilbert-base-uncased-finetuned-conll03-english + --model_name_or_path hf-internal-testing/tiny-random-RobertaForTokenClassification --dataset_name conll2003 --apply_quantization --apply_pruning @@ -132,16 +129,12 @@ def test_run_ner(self): with patch.object(sys, "argv", test_args): run_ner.main() results = get_results(tmp_dir) - self.assertGreaterEqual(results["eval_accuracy"], 0.70) - self.assertGreaterEqual(results["eval_f1"], 0.70) - self.assertGreaterEqual(results["eval_precision"], 0.70) - self.assertGreaterEqual(results["eval_recall"], 0.70) def test_run_swag(self): with tempfile.TemporaryDirectory() as tmp_dir: test_args = f""" run_swag.py - --model_name_or_path ehdwns1516/bert-base-uncased_SWAG + --model_name_or_path hf-internal-testing/tiny-random-AlbertForMultipleChoice --apply_quantization --apply_pruning --target_sparsity 0.02 @@ -160,7 +153,6 @@ def test_run_swag(self): with patch.object(sys, "argv", test_args): run_swag.main() results = get_results(tmp_dir) - self.assertGreaterEqual(results["eval_accuracy"], 0.60) def test_run_clm(self): quantization_approach = "dynamic" @@ -168,7 +160,7 @@ def test_run_clm(self): with tempfile.TemporaryDirectory() as tmp_dir: test_args = f""" run_clm.py - --model_name_or_path EleutherAI/gpt-neo-125M + --model_name_or_path hf-internal-testing/tiny-random-GPT2LMHeadModel --dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --apply_quantization @@ -190,7 +182,6 @@ def test_run_clm(self): with patch.object(sys, "argv", test_args): run_clm.main() results = get_results(tmp_dir) - self.assertLessEqual(results["eval_loss"], 15) def test_run_mlm(self): quantization_approach = "static" @@ -198,7 +189,7 @@ def test_run_mlm(self): with tempfile.TemporaryDirectory() as tmp_dir: test_args = f""" run_mlm.py - --model_name_or_path google/electra-small-discriminator + --model_name_or_path hf-internal-testing/tiny-random-DistilBertForMaskedLM --dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --apply_quantization @@ -220,7 +211,6 @@ def test_run_mlm(self): with patch.object(sys, "argv", test_args): run_mlm.main() results = get_results(tmp_dir) - self.assertLessEqual(results["eval_loss"], 15) if __name__ == "__main__": From ba0bb81328b019dfe29220ac5a5f93f5773449a0 Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Tue, 19 Mar 2024 15:28:40 +0100 Subject: [PATCH 2/4] remove ipex extra --- .github/workflows/test_inc_examples.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_inc_examples.yml b/.github/workflows/test_inc_examples.yml index a5c735cb14..2bfb77a769 100644 --- a/.github/workflows/test_inc_examples.yml +++ b/.github/workflows/test_inc_examples.yml @@ -35,7 +35,7 @@ jobs: - name: Install dependencies run: | - pip install optimum[neural-compressor,ipex] pytest + pip install optimum[neural-compressor] pytest pip install -r examples/neural_compressor/text-classification/requirements.txt pip install -r examples/neural_compressor/question-answering/requirements.txt pip install -r examples/neural_compressor/token-classification/requirements.txt @@ -44,4 +44,4 @@ jobs: - name: Test examples run: | - python -m pytest examples/neural_compressor/test_examples.py \ No newline at end of file + python -m pytest examples/neural_compressor/test_examples.py From e757ac2acd8807a20b2e88082261bfd9e67c6d49 Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Tue, 19 Mar 2024 15:47:15 +0100 Subject: [PATCH 3/4] fix test pytest version --- .github/workflows/test_inc_examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_inc_examples.yml b/.github/workflows/test_inc_examples.yml index 2bfb77a769..8e80474d98 100644 --- a/.github/workflows/test_inc_examples.yml +++ b/.github/workflows/test_inc_examples.yml @@ -35,7 +35,7 @@ jobs: - name: Install dependencies run: | - pip install optimum[neural-compressor] pytest + pip install optimum[neural-compressor] pytest==8.0.0 pip install -r examples/neural_compressor/text-classification/requirements.txt pip install -r examples/neural_compressor/question-answering/requirements.txt pip install -r examples/neural_compressor/token-classification/requirements.txt From 0a22e7f16a174d22f4b484b8a540abf7642920a7 Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Tue, 19 Mar 2024 15:50:37 +0100 Subject: [PATCH 4/4] fix style --- examples/neural_compressor/test_examples.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/neural_compressor/test_examples.py b/examples/neural_compressor/test_examples.py index b4be6064b2..f7e91054e9 100644 --- a/examples/neural_compressor/test_examples.py +++ b/examples/neural_compressor/test_examples.py @@ -78,7 +78,7 @@ def test_run_glue(self): with patch.object(sys, "argv", test_args): run_glue.main() - results = get_results(tmp_dir) + get_results(tmp_dir) def test_run_qa(self): with tempfile.TemporaryDirectory() as tmp_dir: @@ -103,7 +103,7 @@ def test_run_qa(self): with patch.object(sys, "argv", test_args): run_qa.main() - results = get_results(tmp_dir) + get_results(tmp_dir) def test_run_ner(self): with tempfile.TemporaryDirectory() as tmp_dir: @@ -128,7 +128,7 @@ def test_run_ner(self): with patch.object(sys, "argv", test_args): run_ner.main() - results = get_results(tmp_dir) + get_results(tmp_dir) def test_run_swag(self): with tempfile.TemporaryDirectory() as tmp_dir: @@ -152,7 +152,7 @@ def test_run_swag(self): with patch.object(sys, "argv", test_args): run_swag.main() - results = get_results(tmp_dir) + get_results(tmp_dir) def test_run_clm(self): quantization_approach = "dynamic" @@ -181,7 +181,7 @@ def test_run_clm(self): with patch.object(sys, "argv", test_args): run_clm.main() - results = get_results(tmp_dir) + get_results(tmp_dir) def test_run_mlm(self): quantization_approach = "static" @@ -210,7 +210,7 @@ def test_run_mlm(self): with patch.object(sys, "argv", test_args): run_mlm.main() - results = get_results(tmp_dir) + get_results(tmp_dir) if __name__ == "__main__":