Tutorial: Testing Strategies

This tutorial teaches you to write comprehensive test suites for Mistaber encodings. You will learn test organization, fixture patterns, parameterized testing, and CI integration to ensure your encodings are correct and maintainable.

Duration: 1.5 hours Difficulty: Intermediate Prerequisites: Tutorial 01: Your First Encoding

Minimum Test Requirements

Every encoding must meet these thresholds:

Test Type	Minimum Count	Purpose
Positive tests	5	Verify rules fire correctly
Negative tests	3	Verify rules don't fire incorrectly
Edge cases	2	Test boundary conditions
Multi-world tests	If applicable	Verify inheritance and overrides
Machloket tests	If applicable	Test both positions

Test File Organization

Directory Structure

tests/
  corpus/
    yd_87/
      __init__.py
      conftest.py           # Shared fixtures for YD 87
      test_beheima_chalav.py    # YD 87:1 tests
      test_of_chalav.py         # YD 87:3 (poultry) tests
      test_chaya_chalav.py      # YD 87:3 (wild animal) tests
      test_fish_dairy.py        # YD 87:3 (fish) machloket tests
    yd_89/
      __init__.py
      conftest.py           # Shared fixtures for YD 89
      test_waiting_time.py  # YD 89:1 tests
  worlds/
    __init__.py
    test_inheritance.py     # World inheritance tests
    test_override.py        # Override mechanism tests
    test_machloket.py       # Machloket detection tests
  integration/
    __init__.py
    test_full_queries.py    # End-to-end query tests

Naming Conventions

Pattern	Example	Contents
test_[topic].py	test_beheima_chalav.py	Tests for a specific halachic topic
test_[siman]_[seif].py	test_87_1.py	Tests for a specific seif
conftest.py	conftest.py	Shared fixtures for the directory

The conftest.py File

Shared fixtures go in conftest.py for automatic discovery by pytest.

Example: YD 87 Fixtures

# tests/corpus/yd_87/conftest.py
"""Shared fixtures for Yoreh Deah 87 (Basar Bechalav) tests."""
import pytest
from pathlib import Path
from mistaber.engine import HsrsEngine


@pytest.fixture(scope="module")
def engine():
    """Shared engine instance for all tests in module."""
    return HsrsEngine(Path("mistaber/ontology"))


@pytest.fixture
def beheima_chalav_mixture():
    """Standard beheima (beef) + chalav (milk) mixture."""
    return """
        mixture(m1).
        food(beef). food_type(beef, beheima).
        food(milk). food_type(milk, chalav).
        contains(m1, beef). contains(m1, milk).
    """


@pytest.fixture
def of_chalav_mixture():
    """Standard of (poultry) + chalav mixture."""
    return """
        mixture(m1).
        food(chicken). food_type(chicken, of).
        food(milk). food_type(milk, chalav).
        contains(m1, chicken). contains(m1, milk).
    """


@pytest.fixture
def chaya_chalav_mixture():
    """Standard chaya (wild animal) + chalav mixture."""
    return """
        mixture(m1).
        food(deer). food_type(deer, chaya).
        food(milk). food_type(milk, chalav).
        contains(m1, deer). contains(m1, milk).
    """


@pytest.fixture
def dag_chalav_mixture():
    """Standard dag (fish) + chalav mixture."""
    return """
        mixture(m1).
        food(salmon). food_type(salmon, dag).
        food(cream). food_type(cream, chalav).
        contains(m1, salmon). contains(m1, cream).
    """


@pytest.fixture
def parve_mixture():
    """Parve-only mixture (no basar bechalav concerns)."""
    return """
        mixture(m1).
        food(carrot). food_type(carrot, parve).
        food(rice). food_type(rice, parve).
        contains(m1, carrot). contains(m1, rice).
    """


@pytest.fixture
def empty_scenario():
    """Empty scenario with no declarations."""
    return ""

Writing Positive Tests

Positive tests verify that rules fire when they should.

Pattern: Arrange-Act-Assert

def test_rule_fires_correctly(self, beheima_chalav_mixture):
    """Beheima+chalav eating is forbidden d'oraita."""
    # Arrange: Set up the scenario
    scenario = beheima_chalav_mixture

    # Act: Run the query
    result = query(scenario, world="mechaber")

    # Assert: Verify expected outcomes
    assert result.holds("issur(achiila, m1, d_oraita)")
    assert result.holds("issur(bishul, m1, d_oraita)")
    assert result.holds("issur(hanaah, m1, d_oraita)")

Complete Positive Test Set

class TestBeheimaChalaviIssur:
    """Positive tests for beheima + chalav prohibition (YD 87:1)."""

    # Test 1: Eating prohibition
    def test_issur_achiila(self, beheima_chalav_mixture):
        """Eating beheima+chalav is forbidden d'oraita."""
        result = query(beheima_chalav_mixture, world="mechaber")
        assert result.holds("issur(achiila, m1, d_oraita)")

    # Test 2: Cooking prohibition
    def test_issur_bishul(self, beheima_chalav_mixture):
        """Cooking beheima+chalav is forbidden d'oraita."""
        result = query(beheima_chalav_mixture, world="mechaber")
        assert result.holds("issur(bishul, m1, d_oraita)")

    # Test 3: Benefit prohibition
    def test_issur_hanaah(self, beheima_chalav_mixture):
        """Benefit from beheima+chalav is forbidden d'oraita."""
        result = query(beheima_chalav_mixture, world="mechaber")
        assert result.holds("issur(hanaah, m1, d_oraita)")

    # Test 4: Madrega classification
    def test_madrega_doraita(self, beheima_chalav_mixture):
        """Beheima+chalav is classified as d'oraita."""
        result = query(beheima_chalav_mixture, world="mechaber")
        assert result.holds("basar_chalav_madrega(m1, d_oraita)")

    # Test 5: Mixture detection
    def test_mixture_detected(self, beheima_chalav_mixture):
        """Beheima+chalav mixture is correctly detected."""
        result = query(beheima_chalav_mixture, world="mechaber")
        assert result.holds("is_beheima_chalav_mixture(m1)")

Writing Negative Tests

Negative tests verify that rules do NOT fire when they shouldn't.

Pattern: Verify Absence

def test_rule_does_not_fire(self, parve_mixture):
    """Parve foods do not trigger basar bechalav."""
    # Arrange
    scenario = parve_mixture

    # Act
    result = query(scenario, world="mechaber")

    # Assert: Rules should NOT fire
    assert not result.holds("issur(achiila, m1, _)")
    assert not result.holds("is_beheima_chalav_mixture(m1)")

Complete Negative Test Set

class TestNoIssurim:
    """Negative tests: Scenarios that should NOT trigger issur."""

    # Test 1: Parve only
    def test_parve_no_issur(self, parve_mixture):
        """Parve-only mixture has no basar bechalav issur."""
        result = query(parve_mixture, world="mechaber")
        assert not result.holds("issur(achiila, m1, _)")
        assert not result.holds("is_beheima_chalav_mixture(m1)")
        assert not result.holds("is_of_chalav_mixture(m1)")

    # Test 2: Single food item
    def test_single_food_no_mixture(self):
        """Single food item is not a basar bechalav mixture."""
        scenario = """
            food(beef). food_type(beef, beheima).
        """
        result = query(scenario, world="mechaber")
        assert not result.holds("mixture(_)")
        assert not result.holds("issur(achiila, _, _)")

    # Test 3: Meat only (no dairy)
    def test_meat_only_no_issur(self):
        """Meat without dairy has no basar bechalav issur."""
        scenario = """
            mixture(m1).
            food(beef). food_type(beef, beheima).
            food(chicken). food_type(chicken, of).
            contains(m1, beef). contains(m1, chicken).
        """
        result = query(scenario, world="mechaber")
        assert not result.holds("is_beheima_chalav_mixture(m1)")
        assert not result.holds("issur(achiila, m1, d_oraita)")

Writing Edge Case Tests

Edge cases test boundary conditions and unusual scenarios.

Identifying Edge Cases

Category	Example
Empty inputs	No foods declared
Single values	Only one food item
Boundary values	Exactly 60:1 ratio for bitul
Unusual combinations	Multiple meat types
Missing data	Mixture without contains

Complete Edge Case Set

class TestEdgeCases:
    """Edge case tests for boundary conditions."""

    # Edge 1: Mixture declared but empty
    def test_empty_mixture(self):
        """Mixture with no contents is not basar bechalav."""
        scenario = """
            mixture(m1).
            % No contains/2 declarations
        """
        result = query(scenario, world="mechaber")
        assert not result.holds("is_beheima_chalav_mixture(m1)")

    # Edge 2: Multiple meat types
    def test_multiple_meat_types(self):
        """Mixture with multiple meat types still triggers."""
        scenario = """
            mixture(m1).
            food(beef). food_type(beef, beheima).
            food(chicken). food_type(chicken, of).
            food(milk). food_type(milk, chalav).
            contains(m1, beef). contains(m1, chicken). contains(m1, milk).
        """
        result = query(scenario, world="mechaber")
        # Both mixture types detected
        assert result.holds("is_beheima_chalav_mixture(m1)")
        assert result.holds("is_of_chalav_mixture(m1)")

    # Edge 3: Food type not declared
    def test_missing_food_type(self):
        """Food without type does not trigger rules."""
        scenario = """
            mixture(m1).
            food(mystery). % No food_type
            food(milk). food_type(milk, chalav).
            contains(m1, mystery). contains(m1, milk).
        """
        result = query(scenario, world="mechaber")
        assert not result.holds("is_beheima_chalav_mixture(m1)")

    # Edge 4: Boundary ratio for bitul
    def test_exact_60_ratio_bitul(self):
        """Exactly 60:1 ratio satisfies bitul requirement."""
        scenario = """
            mixture(m1).
            issur_component(m1, i1).
            heter_component(m1, h1).
            ratio(h1, i1, 60).  % Exactly 60
        """
        result = query(scenario, world="mechaber")
        assert result.holds("batel(m1)")

Multi-World Testing

Multi-world tests verify inheritance and override behavior.

Testing Inheritance

class TestWorldInheritance:
    """Tests for world inheritance behavior."""

    @pytest.fixture
    def standard_scenario(self):
        return """
            mixture(m1).
            food(beef). food_type(beef, beheima).
            food(milk). food_type(milk, chalav).
            contains(m1, beef). contains(m1, milk).
        """

    def test_sefardi_yo_inherits_from_mechaber(self, standard_scenario):
        """sefardi_yo inherits mechaber's rulings."""
        mechaber = query(standard_scenario, world="mechaber")
        sefardi = query(standard_scenario, world="sefardi_yo")

        # Same issur in both
        assert mechaber.holds("issur(achiila, m1, d_oraita)")
        assert sefardi.holds("issur(achiila, m1, d_oraita)")

    def test_ashk_mb_inherits_from_rema(self, standard_scenario):
        """ashk_mb inherits rema's rulings."""
        rema = query(standard_scenario, world="rema")
        ashk_mb = query(standard_scenario, world="ashk_mb")

        # Same issur in both
        assert rema.holds("issur(achiila, m1, d_oraita)")
        assert ashk_mb.holds("issur(achiila, m1, d_oraita)")

    def test_all_worlds_agree_on_beheima(self, standard_scenario):
        """All worlds agree beheima+chalav is d'oraita."""
        worlds = ["mechaber", "rema", "gra", "sefardi_yo", "ashk_mb", "ashk_ah"]

        for world in worlds:
            result = query(standard_scenario, world=world)
            assert result.holds("issur(achiila, m1, d_oraita)"), \
                f"{world} should have issur d'oraita"

Testing Overrides

class TestOverrideBehavior:
    """Tests for override mechanism in machloket."""

    @pytest.fixture
    def fish_dairy(self):
        return """
            mixture(m1).
            food(salmon). food_type(salmon, dag).
            food(cream). food_type(cream, chalav).
            contains(m1, salmon). contains(m1, cream).
        """

    def test_override_blocks_inheritance(self, fish_dairy):
        """Rema's override blocks mechaber's sakana."""
        mechaber = query(fish_dairy, world="mechaber")
        rema = query(fish_dairy, world="rema")

        assert mechaber.holds("sakana(m1)")
        assert not rema.holds("sakana(m1)")

    def test_override_propagates_to_children(self, fish_dairy):
        """Override propagates to child worlds."""
        rema = query(fish_dairy, world="rema")
        ashk_mb = query(fish_dairy, world="ashk_mb")

        # Both should NOT have sakana
        assert rema.holds("no_sakana(m1)")
        assert ashk_mb.holds("no_sakana(m1)")

        # ashk_mb should NOT inherit mechaber's sakana
        assert not ashk_mb.holds("sakana(m1)")

Parameterized Testing

Use @pytest.mark.parametrize for testing multiple cases efficiently.

Parameterized World Tests

@pytest.mark.parametrize("world,expected_sakana", [
    ("mechaber", True),
    ("sefardi_yo", True),
    ("rema", False),
    ("ashk_mb", False),
    ("ashk_ah", False),
])
def test_fish_dairy_by_world(dag_chalav_mixture, world, expected_sakana):
    """Fish+dairy sakana status varies by world."""
    result = query(dag_chalav_mixture, world=world)

    if expected_sakana:
        assert result.holds("sakana(m1)"), f"{world} should have sakana"
    else:
        assert result.holds("no_sakana(m1)") or not result.holds("sakana(m1)"), \
            f"{world} should NOT have sakana"

Parameterized Scenario Tests

@pytest.mark.parametrize("meat_type,expected_madrega", [
    ("beheima", "d_oraita"),
    ("chaya", "d_rabanan"),
    ("of", "d_rabanan"),
])
def test_madrega_by_meat_type(meat_type, expected_madrega):
    """Madrega varies by meat type."""
    scenario = f"""
        mixture(m1).
        food(meat). food_type(meat, {meat_type}).
        food(milk). food_type(milk, chalav).
        contains(m1, meat). contains(m1, milk).
    """
    result = query(scenario, world="mechaber")
    assert result.holds(f"basar_chalav_madrega(m1, {expected_madrega})")

Test Markers

Use markers to categorize tests for selective running.

Defining Markers

# pytest.ini
[pytest]
markers =
    multiworld: tests that verify behavior across multiple worlds
    machloket: tests for disputed rulings
    doraita: tests for Torah-level prohibitions
    drabanan: tests for rabbinic prohibitions
    slow: tests that take longer to run
    integration: end-to-end integration tests

Using Markers

@pytest.mark.multiworld
def test_inheritance_all_worlds():
    """Test requires checking multiple worlds."""
    pass

@pytest.mark.machloket
def test_fish_dairy_dispute():
    """Test for machloket between Mechaber and Rema."""
    pass

@pytest.mark.doraita
@pytest.mark.multiworld
def test_beheima_chalav_all_worlds():
    """D'oraita test across all worlds."""
    pass

Running Marked Tests

# Run only multi-world tests
pytest -m multiworld

# Run machloket tests
pytest -m machloket

# Run d'oraita tests (not slow)
pytest -m "doraita and not slow"

CI Integration

GitHub Actions Workflow

# .github/workflows/test.yml
name: Tests

on: [push, pull_request]

jobs:
  test:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: |
          pip install -e ".[dev]"
          pip install pytest pytest-cov

      - name: Run tests
        run: pytest tests/ --cov=mistaber --cov-report=xml

      - name: Check coverage threshold
        run: coverage report --fail-under=80

      - name: Upload coverage
        uses: codecov/codecov-action@v3
        with:
          files: coverage.xml

Pre-Commit Hook

# .pre-commit-config.yaml
repos:
  - repo: local
    hooks:
      - id: pytest-check
        name: pytest
        entry: pytest tests/ -x -q --tb=short
        language: system
        pass_filenames: false
        always_run: true

Test Checklist

Before submitting your encoding:

• [ ] At least 5 positive test cases
• [ ] At least 3 negative test cases
• [ ] At least 2 edge cases
• [ ] Multi-world tests for inheritance (if applicable)
• [ ] Machloket tests for both positions (if applicable)
• [ ] All tests pass locally (pytest tests/)
• [ ] Tests organized in correct directory
• [ ] Fixtures shared in conftest.py
• [ ] Test names are descriptive
• [ ] Markers applied where appropriate

Common Testing Mistakes

Mistake 1: Testing Implementation, Not Behavior

# WRONG - tests internal predicates
def test_internal_helper():
    result = query(scenario, world="mechaber")
    assert result.holds("_helper_predicate(x)")

# CORRECT - tests observable behavior
def test_observable_ruling():
    result = query(scenario, world="mechaber")
    assert result.holds("issur(achiila, m1, d_oraita)")

Mistake 2: Insufficient World Coverage

# WRONG - only tests one world
def test_ruling():
    result = query(scenario, world="mechaber")
    assert result.holds("ruling(x)")

# CORRECT - tests inheritance chain
def test_ruling_with_inheritance():
    mechaber = query(scenario, world="mechaber")
    sefardi = query(scenario, world="sefardi_yo")
    assert mechaber.holds("ruling(x)")
    assert sefardi.holds("ruling(x)")  # Verify inheritance

Mistake 3: Brittle Assertions

# WRONG - depends on exact atom format
def test_brittle():
    result = query(scenario, world="mechaber")
    assert "issur(achiila,m1,d_oraita)" in result.raw_output

# CORRECT - uses semantic assertion
def test_robust():
    result = query(scenario, world="mechaber")
    assert result.holds("issur(achiila, m1, d_oraita)")

Summary

Effective testing requires:

1. Organized structure: Tests in logical directories with shared fixtures
2. Complete coverage: Positive, negative, edge, and multi-world tests
3. Parameterization: Efficient testing of multiple cases
4. CI integration: Automated testing on every commit
5. Clear assertions: Test behavior, not implementation

Next Steps

Continue with:

1. Tutorial 05: Review Process - Navigate reviews

Quick Reference: Test Patterns

# === Positive test ===
def test_rule_fires(fixture):
    result = query(fixture, world="world")
    assert result.holds("expected_predicate(args)")

# === Negative test ===
def test_rule_does_not_fire(fixture):
    result = query(fixture, world="world")
    assert not result.holds("unexpected_predicate(_)")

# === Parameterized test ===
@pytest.mark.parametrize("input,expected", [...])
def test_parameterized(input, expected):
    result = query(input, world="world")
    assert result.holds(expected)

# === Multi-world test ===
def test_inheritance():
    parent = query(scenario, world="parent")
    child = query(scenario, world="child")
    assert parent.holds("predicate(x)")
    assert child.holds("predicate(x)")