Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add test for INC examples #617

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

Add test for INC examples #617

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
404ec40
Add test for INC examples
echarlaix Mar 19, 2024
ba0bb81
remove ipex extra
echarlaix Mar 19, 2024
e757ac2
fix test pytest version
echarlaix Mar 19, 2024
0a22e7f
fix style
echarlaix Mar 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions .github/workflows/test_inc_examples.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
name: INC - Examples Test

on:
workflow_dispatch:
schedule:
- cron: 0 1 * * 1 # run weekly: every Monday at 1am
push:
paths:
- '.github/workflows/test_inc_examples.yml'
- 'examples/neural_compressor/*'
pull_request:
paths:
- '.github/workflows/test_inc_examples.yml'
- 'examples/neural_compressor/*'

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

jobs:
build:
strategy:
fail-fast: false
matrix:
python-version: ["3.8", "3.10"]

runs-on: ubuntu-20.04

steps:
- uses: actions/checkout@v2
- name: Setup Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}

- name: Install dependencies
run: |
pip install optimum[neural-compressor] pytest==8.0.0
pip install -r examples/neural_compressor/text-classification/requirements.txt
pip install -r examples/neural_compressor/question-answering/requirements.txt
pip install -r examples/neural_compressor/token-classification/requirements.txt
pip install -r examples/neural_compressor/language-modeling/requirements.txt
pip install -r examples/neural_compressor/multiple-choice/requirements.txt

- name: Test examples
run: |
python -m pytest examples/neural_compressor/test_examples.py
34 changes: 12 additions & 22 deletions examples/neural_compressor/test_examples.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def test_run_glue(self):
with tempfile.TemporaryDirectory() as tmp_dir:
test_args = f"""
run_glue.py
--model_name_or_path distilbert-base-uncased-finetuned-sst-2-english
--model_name_or_path hf-internal-testing/tiny-random-DistilBertForSequenceClassification
--task_name sst2
--apply_quantization
--apply_pruning
Expand All @@ -78,14 +78,13 @@ def test_run_glue(self):

with patch.object(sys, "argv", test_args):
run_glue.main()
results = get_results(tmp_dir)
self.assertGreaterEqual(results["eval_accuracy"], 0.70)
get_results(tmp_dir)

def test_run_qa(self):
with tempfile.TemporaryDirectory() as tmp_dir:
test_args = f"""
run_qa.py
--model_name_or_path distilbert-base-uncased-distilled-squad
--model_name_or_path hf-internal-testing/tiny-random-DistilBertForQuestionAnswering
--dataset_name squad
--apply_quantization
--apply_pruning
Expand All @@ -104,15 +103,13 @@ def test_run_qa(self):

with patch.object(sys, "argv", test_args):
run_qa.main()
results = get_results(tmp_dir)
self.assertGreaterEqual(results["eval_f1"], 70)
self.assertGreaterEqual(results["eval_exact_match"], 70)
get_results(tmp_dir)

def test_run_ner(self):
with tempfile.TemporaryDirectory() as tmp_dir:
test_args = f"""
run_ner.py
--model_name_or_path elastic/distilbert-base-uncased-finetuned-conll03-english
--model_name_or_path hf-internal-testing/tiny-random-RobertaForTokenClassification
--dataset_name conll2003
--apply_quantization
--apply_pruning
Expand All @@ -131,17 +128,13 @@ def test_run_ner(self):

with patch.object(sys, "argv", test_args):
run_ner.main()
results = get_results(tmp_dir)
self.assertGreaterEqual(results["eval_accuracy"], 0.70)
self.assertGreaterEqual(results["eval_f1"], 0.70)
self.assertGreaterEqual(results["eval_precision"], 0.70)
self.assertGreaterEqual(results["eval_recall"], 0.70)
get_results(tmp_dir)

def test_run_swag(self):
with tempfile.TemporaryDirectory() as tmp_dir:
test_args = f"""
run_swag.py
--model_name_or_path ehdwns1516/bert-base-uncased_SWAG
--model_name_or_path hf-internal-testing/tiny-random-AlbertForMultipleChoice
--apply_quantization
--apply_pruning
--target_sparsity 0.02
Expand All @@ -159,16 +152,15 @@ def test_run_swag(self):

with patch.object(sys, "argv", test_args):
run_swag.main()
results = get_results(tmp_dir)
self.assertGreaterEqual(results["eval_accuracy"], 0.60)
get_results(tmp_dir)

def test_run_clm(self):
quantization_approach = "dynamic"

with tempfile.TemporaryDirectory() as tmp_dir:
test_args = f"""
run_clm.py
--model_name_or_path EleutherAI/gpt-neo-125M
--model_name_or_path hf-internal-testing/tiny-random-GPT2LMHeadModel
--dataset_name wikitext
--dataset_config_name wikitext-2-raw-v1
--apply_quantization
Expand All @@ -189,16 +181,15 @@ def test_run_clm(self):

with patch.object(sys, "argv", test_args):
run_clm.main()
results = get_results(tmp_dir)
self.assertLessEqual(results["eval_loss"], 15)
get_results(tmp_dir)

def test_run_mlm(self):
quantization_approach = "static"

with tempfile.TemporaryDirectory() as tmp_dir:
test_args = f"""
run_mlm.py
--model_name_or_path google/electra-small-discriminator
--model_name_or_path hf-internal-testing/tiny-random-DistilBertForMaskedLM
--dataset_name wikitext
--dataset_config_name wikitext-2-raw-v1
--apply_quantization
Expand All @@ -219,8 +210,7 @@ def test_run_mlm(self):

with patch.object(sys, "argv", test_args):
run_mlm.main()
results = get_results(tmp_dir)
self.assertLessEqual(results["eval_loss"], 15)
get_results(tmp_dir)


if __name__ == "__main__":
Expand Down
Loading