diff --git a/brainscore_language/models/falcon7b/__init__.py b/brainscore_language/models/falcon7b/__init__.py new file mode 100644 index 00000000..ea5851f8 --- /dev/null +++ b/brainscore_language/models/falcon7b/__init__.py @@ -0,0 +1,12 @@ +from brainscore_language import model_registry +from brainscore_language import ArtificialSubject +from brainscore_language.model_helpers.huggingface import HuggingfaceSubject + +# Falcon-7B: 32 transformer layers, hidden size 4544. +# Layer 31 (last) chosen as default mapping pending benchmark-driven selection. +model_registry['falcon-7b'] = lambda: HuggingfaceSubject( + model_id='tiiuae/falcon-7b', + region_layer_mapping={ + ArtificialSubject.RecordingTarget.language_system: 'transformer.h.31' + }, +) diff --git a/brainscore_language/models/falcon7b/metadata.yml b/brainscore_language/models/falcon7b/metadata.yml new file mode 100644 index 00000000..80dc8db8 --- /dev/null +++ b/brainscore_language/models/falcon7b/metadata.yml @@ -0,0 +1,15 @@ +models: + falcon7b: + architecture: DCNN + model_family: falcon7b + total_parameter_count: 1234567 + trainable_parameter_count: 1234567 + total_layers: 55 + trainable_layers: 40 + model_size_mb: 1202 + training_dataset: null + task_specialization: null + brainscore_link: https://github.com/brain-score/language/tree/master/brainscore_language/models/falcon7b + huggingface_link: null + extra_notes: Temporary hardcoded metadata - will be replaced with actual generation + runnable: true diff --git a/brainscore_language/models/falcon7b/test.py b/brainscore_language/models/falcon7b/test.py new file mode 100644 index 00000000..b721f655 --- /dev/null +++ b/brainscore_language/models/falcon7b/test.py @@ -0,0 +1,37 @@ +import numpy as np +import pytest + +from brainscore_language import load_model +from brainscore_language.artificial_subject import ArtificialSubject + + +@pytest.mark.memory_intense +def test_load_model(): + model = load_model('falcon-7b') + assert model is not None + + +@pytest.mark.memory_intense +def test_neural(): + model = load_model('falcon-7b') + text = ['the quick brown fox', 'jumps over', 'the lazy dog'] + model.start_neural_recording( + recording_target=ArtificialSubject.RecordingTarget.language_system, + recording_type=ArtificialSubject.RecordingType.fMRI, + ) + representations = model.digest_text(text)['neural'] + assert len(representations['presentation']) == 3 + np.testing.assert_array_equal(representations['stimulus'], text) + assert len(representations['neuroid']) == 4544 + + +@pytest.mark.memory_intense +def test_next_word(): + model = load_model('falcon-7b') + text = ['the quick brown fox', 'jumps over', 'the lazy'] + model.start_behavioral_task(task=ArtificialSubject.Task.next_word) + next_words = model.digest_text(text)['behavior'] + assert len(next_words) == 3 + for word in next_words.values: + assert isinstance(word, str) + assert len(word.strip()) > 0 diff --git a/brainscore_language/models/opt6_7b/__init__.py b/brainscore_language/models/opt6_7b/__init__.py new file mode 100644 index 00000000..69abbd94 --- /dev/null +++ b/brainscore_language/models/opt6_7b/__init__.py @@ -0,0 +1,12 @@ +from brainscore_language import model_registry +from brainscore_language import ArtificialSubject +from brainscore_language.model_helpers.huggingface import HuggingfaceSubject + +# OPT-6.7B: 32 transformer layers, hidden size 4096. +# Layer 31 (last) chosen as default mapping pending benchmark-driven selection. +model_registry['opt-6.7b'] = lambda: HuggingfaceSubject( + model_id='facebook/opt-6.7b', + region_layer_mapping={ + ArtificialSubject.RecordingTarget.language_system: 'model.decoder.layers.31' + }, +) diff --git a/brainscore_language/models/opt6_7b/metadata.yml b/brainscore_language/models/opt6_7b/metadata.yml new file mode 100644 index 00000000..12aee46d --- /dev/null +++ b/brainscore_language/models/opt6_7b/metadata.yml @@ -0,0 +1,15 @@ +models: + opt6_7b: + architecture: DCNN + model_family: opt6_7b + total_parameter_count: 1234567 + trainable_parameter_count: 1234567 + total_layers: 55 + trainable_layers: 40 + model_size_mb: 1202 + training_dataset: null + task_specialization: null + brainscore_link: https://github.com/brain-score/language/tree/master/brainscore_language/models/opt6_7b + huggingface_link: null + extra_notes: Temporary hardcoded metadata - will be replaced with actual generation + runnable: true diff --git a/brainscore_language/models/opt6_7b/test.py b/brainscore_language/models/opt6_7b/test.py new file mode 100644 index 00000000..a26dd11c --- /dev/null +++ b/brainscore_language/models/opt6_7b/test.py @@ -0,0 +1,37 @@ +import numpy as np +import pytest + +from brainscore_language import load_model +from brainscore_language.artificial_subject import ArtificialSubject + + +@pytest.mark.memory_intense +def test_load_model(): + model = load_model('opt-6.7b') + assert model is not None + + +@pytest.mark.memory_intense +def test_neural(): + model = load_model('opt-6.7b') + text = ['the quick brown fox', 'jumps over', 'the lazy dog'] + model.start_neural_recording( + recording_target=ArtificialSubject.RecordingTarget.language_system, + recording_type=ArtificialSubject.RecordingType.fMRI, + ) + representations = model.digest_text(text)['neural'] + assert len(representations['presentation']) == 3 + np.testing.assert_array_equal(representations['stimulus'], text) + assert len(representations['neuroid']) == 4096 + + +@pytest.mark.memory_intense +def test_next_word(): + model = load_model('opt-6.7b') + text = ['the quick brown fox', 'jumps over', 'the lazy'] + model.start_behavioral_task(task=ArtificialSubject.Task.next_word) + next_words = model.digest_text(text)['behavior'] + assert len(next_words) == 3 + for word in next_words.values: + assert isinstance(word, str) + assert len(word.strip()) > 0 diff --git a/brainscore_language/models/pythia12b/__init__.py b/brainscore_language/models/pythia12b/__init__.py new file mode 100644 index 00000000..01590aa7 --- /dev/null +++ b/brainscore_language/models/pythia12b/__init__.py @@ -0,0 +1,12 @@ +from brainscore_language import model_registry +from brainscore_language import ArtificialSubject +from brainscore_language.model_helpers.huggingface import HuggingfaceSubject + +# Pythia-12B: 36 transformer layers, hidden size 5120. +# Layer 35 (last) chosen as default mapping pending benchmark-driven selection. +model_registry['pythia-12b'] = lambda: HuggingfaceSubject( + model_id='EleutherAI/pythia-12b', + region_layer_mapping={ + ArtificialSubject.RecordingTarget.language_system: 'gpt_neox.layers.35' + }, +) diff --git a/brainscore_language/models/pythia12b/metadata.yml b/brainscore_language/models/pythia12b/metadata.yml new file mode 100644 index 00000000..e5230294 --- /dev/null +++ b/brainscore_language/models/pythia12b/metadata.yml @@ -0,0 +1,15 @@ +models: + pythia12b: + architecture: DCNN + model_family: pythia12b + total_parameter_count: 1234567 + trainable_parameter_count: 1234567 + total_layers: 55 + trainable_layers: 40 + model_size_mb: 1202 + training_dataset: null + task_specialization: null + brainscore_link: https://github.com/brain-score/language/tree/master/brainscore_language/models/pythia12b + huggingface_link: null + extra_notes: Temporary hardcoded metadata - will be replaced with actual generation + runnable: true diff --git a/brainscore_language/models/pythia12b/test.py b/brainscore_language/models/pythia12b/test.py new file mode 100644 index 00000000..07492a64 --- /dev/null +++ b/brainscore_language/models/pythia12b/test.py @@ -0,0 +1,37 @@ +import numpy as np +import pytest + +from brainscore_language import load_model +from brainscore_language.artificial_subject import ArtificialSubject + + +@pytest.mark.memory_intense +def test_load_model(): + model = load_model('pythia-12b') + assert model is not None + + +@pytest.mark.memory_intense +def test_neural(): + model = load_model('pythia-12b') + text = ['the quick brown fox', 'jumps over', 'the lazy dog'] + model.start_neural_recording( + recording_target=ArtificialSubject.RecordingTarget.language_system, + recording_type=ArtificialSubject.RecordingType.fMRI, + ) + representations = model.digest_text(text)['neural'] + assert len(representations['presentation']) == 3 + np.testing.assert_array_equal(representations['stimulus'], text) + assert len(representations['neuroid']) == 5120 + + +@pytest.mark.memory_intense +def test_next_word(): + model = load_model('pythia-12b') + text = ['the quick brown fox', 'jumps over', 'the lazy'] + model.start_behavioral_task(task=ArtificialSubject.Task.next_word) + next_words = model.digest_text(text)['behavior'] + assert len(next_words) == 3 + for word in next_words.values: + assert isinstance(word, str) + assert len(word.strip()) > 0 diff --git a/brainscore_language/models/qwen2_5_3b/__init__.py b/brainscore_language/models/qwen2_5_3b/__init__.py new file mode 100644 index 00000000..5fb04086 --- /dev/null +++ b/brainscore_language/models/qwen2_5_3b/__init__.py @@ -0,0 +1,12 @@ +from brainscore_language import model_registry +from brainscore_language import ArtificialSubject +from brainscore_language.model_helpers.huggingface import HuggingfaceSubject + +# Qwen2.5-3B: 36 transformer layers, hidden size 2048. +# Layer 35 (last) chosen as default mapping pending benchmark-driven selection. +model_registry['qwen2.5-3b'] = lambda: HuggingfaceSubject( + model_id='Qwen/Qwen2.5-3B', + region_layer_mapping={ + ArtificialSubject.RecordingTarget.language_system: 'model.layers.35' + }, +) diff --git a/brainscore_language/models/qwen2_5_3b/metadata.yml b/brainscore_language/models/qwen2_5_3b/metadata.yml new file mode 100644 index 00000000..e7b4d9b8 --- /dev/null +++ b/brainscore_language/models/qwen2_5_3b/metadata.yml @@ -0,0 +1,15 @@ +models: + qwen2_5_3b: + architecture: DCNN + model_family: qwen2_5_3b + total_parameter_count: 1234567 + trainable_parameter_count: 1234567 + total_layers: 55 + trainable_layers: 40 + model_size_mb: 1202 + training_dataset: null + task_specialization: null + brainscore_link: https://github.com/brain-score/language/tree/master/brainscore_language/models/qwen2_5_3b + huggingface_link: null + extra_notes: Temporary hardcoded metadata - will be replaced with actual generation + runnable: true diff --git a/brainscore_language/models/qwen2_5_3b/test.py b/brainscore_language/models/qwen2_5_3b/test.py new file mode 100644 index 00000000..3c32baba --- /dev/null +++ b/brainscore_language/models/qwen2_5_3b/test.py @@ -0,0 +1,37 @@ +import numpy as np +import pytest + +from brainscore_language import load_model +from brainscore_language.artificial_subject import ArtificialSubject + + +@pytest.mark.memory_intense +def test_load_model(): + model = load_model('qwen2.5-3b') + assert model is not None + + +@pytest.mark.memory_intense +def test_neural(): + model = load_model('qwen2.5-3b') + text = ['the quick brown fox', 'jumps over', 'the lazy dog'] + model.start_neural_recording( + recording_target=ArtificialSubject.RecordingTarget.language_system, + recording_type=ArtificialSubject.RecordingType.fMRI, + ) + representations = model.digest_text(text)['neural'] + assert len(representations['presentation']) == 3 + np.testing.assert_array_equal(representations['stimulus'], text) + assert len(representations['neuroid']) == 2048 + + +@pytest.mark.memory_intense +def test_next_word(): + model = load_model('qwen2.5-3b') + text = ['the quick brown fox', 'jumps over', 'the lazy'] + model.start_behavioral_task(task=ArtificialSubject.Task.next_word) + next_words = model.digest_text(text)['behavior'] + assert len(next_words) == 3 + for word in next_words.values: + assert isinstance(word, str) + assert len(word.strip()) > 0