diff --git a/.github/workflows/miss_hit.cfg b/.github/workflows/miss_hit.cfg new file mode 100644 index 0000000..d02fa5c --- /dev/null +++ b/.github/workflows/miss_hit.cfg @@ -0,0 +1,23 @@ +# miss_hit configuration +# for matlab linting and static analysis +# style guide (https://florianschanda.github.io/miss_hit/style_checker.html) +# metrics limit for the code quality (https://florianschanda.github.io/miss_hit/metrics.html) + +project_root + +octave: false + +line_length: 100 + +suppress_rule: "copyright_notice" + +tab_width: 2 + +# snake_case +regex_script_name: "[a-zA-Z]+(_[a-zA-Z0-9]*)*" +regex_function_name: "[a-zA-Z]+(_[a-zA-Z0-9]*)*" + +metric "cnest": limit 4 +metric "file_length": limit 500 +metric "cyc": limit 10 +metric "parameters": limit 5 diff --git a/.github/workflows/run_demos_ci.m b/.github/workflows/run_demos_ci.m new file mode 100644 index 0000000..704836c --- /dev/null +++ b/.github/workflows/run_demos_ci.m @@ -0,0 +1,22 @@ +% run demos with moxunit in github CI + +root_dir = getenv('GITHUB_WORKSPACE'); + +% MOxUnit and MOcov need to be in the matlab path +addpath(fullfile(root_dir, 'MOcov', 'MOcov')); +cd(fullfile(root_dir, 'MOxUnit', 'MOxUnit')); +run moxunit_set_path(); + +% add glm single to path +cd(root_dir); +setup(); + +this_folder = fileparts(mfilename('fullpath')); +test_folder = this_folder; +success = moxunit_runtests(test_folder, '-verbose', '-recursive'); + +if success + system('echo 0 > test_report.log'); +else + system('echo 1 > test_report.log'); +end diff --git a/.github/workflows/run_demos_matlab.yaml b/.github/workflows/run_demos_matlab.yaml new file mode 100644 index 0000000..7aa9ee3 --- /dev/null +++ b/.github/workflows/run_demos_matlab.yaml @@ -0,0 +1,54 @@ +name: MATLAB demos + +# Uses the cron schedule for github actions +# +# will run at 00H00 run on the 1 and 15 of every month +# +# https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows#scheduled-events +# +# ┌───────────── minute (0 - 59) +# │ ┌───────────── hour (0 - 23) +# │ │ ┌───────────── day of the month (1 - 31) +# │ │ │ ┌───────────── month (1 - 12 or JAN-DEC): * means all +# │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT): * means all +# │ │ │ │ │ +# │ │ │ │ │ +# │ │ │ │ │ +# +# - cron "0 0 1,15 * *" + +on: + schedule: + - cron: "0 0 1,15 * *" + +jobs: + demos: + runs-on: ubuntu-20.04 + + steps: + - name: Install MATLAB + uses: matlab-actions/setup-matlab@v1.0.1 + with: + release: R2020a + + - name: Shallow clone GLMsingle + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 # 0 means we only get the last commit, not the whole git history + + - name: Install Moxunit and MOcov + run: | + git clone https://github.com/MOxUnit/MOxUnit.git --depth 1 + git clone https://github.com/MOcov/MOcov.git --depth 1 + + - name: Run commands + uses: matlab-actions/run-command@v1.0.1 + with: + command: + cd(fullfile(getenv('GITHUB_WORKSPACE'), '.github', 'workflows')); + run run_demos_ci; + + - name: Check logs + run: | + cat test_report.log | grep 0 \ No newline at end of file diff --git a/.github/workflows/run_tests_ci.m b/.github/workflows/run_tests_ci.m new file mode 100644 index 0000000..ecf8798 --- /dev/null +++ b/.github/workflows/run_tests_ci.m @@ -0,0 +1,11 @@ +% run tests with code coverage via the run_tests scripts in the root folder. + +root_dir = getenv('GITHUB_WORKSPACE'); + +% MOxUnit and MOcov need to be in the matlab path +addpath(fullfile(root_dir, 'MOcov', 'MOcov')); +cd(fullfile(root_dir, 'MOxUnit', 'MOxUnit')); +run moxunit_set_path(); + +cd(root_dir); +run run_tests(); diff --git a/.github/workflows/run_tests_matlab.yaml b/.github/workflows/run_tests_matlab.yaml new file mode 100644 index 0000000..544ad9f --- /dev/null +++ b/.github/workflows/run_tests_matlab.yaml @@ -0,0 +1,68 @@ +name: MATLAB tests + +# Installs +# - MATLAB github action +# - MOXunit +# - MOcov +# Get test data +# cd into .github/workflows +# run .github/workflows/tests_matlab.m +# If tests pass, uploads coverage to codecov + +on: + push: + # TODO only run on master branch on push + branches: ["main"] + pull_request: + branches: ["*"] + +jobs: + tests: + + strategy: + matrix: + os: [ubuntu-latest] # "macos-latest" or "windows-latest" don't work (yet?) + matlab-version: ["R2020a"] # add more versions if needed: "R2021a" + fail-fast: false # Don't cancel all jobs if one fails + + runs-on: ${{ matrix.os }} + + steps: + - name: Install MATLAB ${{ matrix.matlab-version }} + uses: matlab-actions/setup-matlab@v1.0.1 + with: + release: ${{ matrix.matlab-version }} + + - name: Shallow clone GLMsingle + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 # 0 means we only get the last commit, not the whole git history + + - name: Install Moxunit and MOcov + run: | + git clone https://github.com/MOxUnit/MOxUnit.git --depth 1 + git clone https://github.com/MOcov/MOcov.git --depth 1 + + - name: Download data + run: make tests/data/nsdcoreexampledataset.mat + + - name: Run tests + uses: matlab-actions/run-command@v1.0.1 + with: + command: + cd(fullfile(getenv('GITHUB_WORKSPACE'), '.github', 'workflows')); + run run_tests_ci; + + - name: Check logs + run: | + cat test_report.log | grep 0 + bash <(curl -s https://codecov.io/bash) + + - name: Code coverage + uses: codecov/codecov-action@v1 + with: + file: coverage.xml # optional + flags: matlab # optional + name: codecov-umbrella # optional + fail_ci_if_error: true # optional (default = false) diff --git a/.github/workflows/test_demos.m b/.github/workflows/test_demos.m new file mode 100644 index 0000000..64e73ba --- /dev/null +++ b/.github/workflows/test_demos.m @@ -0,0 +1,32 @@ +function test_suite = test_demos %#ok<*STOUT> + + try % assignment of 'localfunctions' is necessary in Matlab >= 2016 + test_functions = localfunctions(); %#ok<*NASGU> + catch % no problem; early Matlab versions can use initTestSuite fine + end + + initTestSuite; + +end + +function test_demo1() + + run(fullfile(root_dir(), 'matlab', 'examples', 'example1')); + +end + +function test_demo2() + + run(fullfile(root_dir(), 'matlab', 'examples', 'example2')); + +end + +function value = root_dir() + + value = getenv('GITHUB_WORKSPACE'); + + if isempty(value) + value = fullfile(fileparts(mfilename('fullpath')), '..', '..'); + end + +end diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index baaa794..94480c4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,16 +2,34 @@ Information for anyone who would like to contribute to this repository. +- [CONTRIBUTING](#contributing) + - [Repository map](#repository-map) + - [Generic](#generic) + - [Makefile](#makefile) + - [pre-commit](#pre-commit) + - [Matlab](#matlab) + - [Style guide](#style-guide) + - [Tests](#tests) + - [Adding new tests](#adding-new-tests) + - [Continuous integration](#continuous-integration) + - [Tests](#tests-1) + - [Demos](#demos) + - [Python](#python) + - [Style guide](#style-guide-1) + - [Tests](#tests-2) + - [Demos](#demos-1) + - [Continuous integration](#continuous-integration-1) + ## Repository map ```bash ├── .git ├── .github │ └── workflows # Github continuous integration set up -├── examples +├── examples # Python demos │ ├── data │ ├── example1outputs -│ ├── example2outputs +│ └── example2outputs ├── glmsingle # Python implementation │ ├── cod │ ├── design @@ -21,11 +39,13 @@ Information for anyone who would like to contribute to this repository. │ ├── ssq │ └── utils ├── matlab # Matlab implementation -│ ├── examples -│ ├── fracridge +│ ├── examples # Matlab demos +│ ├── fracridge # Fracridge submodule │ └── utilities └── tests # Python and Matlab tests - └── data + ├── data # Data used as inputs for the tests + └── expected # Expected results of the tests +│ └── matlab ``` @@ -33,18 +53,188 @@ Information for anyone who would like to contribute to this repository. ### Makefile +A `Makefile` is used to help set / and automate some things. + +In a terminal type `make help` to see what some of the different "recipes" you +can run with this `Makefile`. + +See +[here for a short intro on using `Makefiles`](https://the-turing-way.netlify.app/reproducible-research/make.html) + ### pre-commit +You can use the [`pre-commit` python package](https://pre-commit.com/) in this +repo to make sure you only commit properly formatted files (for example `.yml` +files). + +1. Install `pre-commit` + +```bash +$ pip3 install pre-commit +``` + +It is also included in `requirements_dev.txt`, so it will installed by running: + +```bash +$ pip3 install -r requirements_dev.txt +``` + +The `.pre-commit-config.yml` file defines the checks to run when committing +files. + +1. Run the following command to install the `pre-commit` "hooks" + +```bash +$ pre-commit install +``` + ## Matlab ### Style guide +The [`miss_hit` python package](https://misshit.org/) is used to help ensure a +consistent coding style for some of the MATLAB code. + +`miss_hit` can check code style, do a certain amount of automatic code +reformating and prevent the code complexity from getting out of hand by running +static code analysis (Static analysis can is a way to measure and track software +quality metrics without additional code like tests). + +`miss_hit` is quite configurable via the use of `miss_hit.cfg` files. + +Install `miss_hit`: + +```bash +$ pip3 install miss_hit +``` + +It is also included in `requirements_dev.txt`, so it will installed by running: + +```bash +$ pip3 install -r requirements_dev.txt +``` + +Style-check your program: + +```bash +$ mh_style --fix path_to_folder_or_m_file +``` + +Make sure your code does not get too complex: + +```bash +$ mh_metric --ci +``` + +You can rule several of those checks by simply typing + +```bash +make lint/miss_hit +``` + ### Tests -#### Demos +For an introduction to testing see +[here](https://the-turing-way.netlify.app/reproducible-research/make.html). + +Running the tests require to have the following toolboxes in your MATLAB path: + +- the [MOxUnit testing framework](https://github.com/MOxUnit/MOxUnit) to run the + tests + ([see installation procedure](https://github.com/MOxUnit/MOxUnit#installation)) +- [MOcov](https://github.com/MOcov/MOcov)) to get a code coverage estimate + ([see installation procedure](https://github.com/MOcov/MOcov#installation)) + +All the tests are in the `tests` folder in files starting with `test_*.m`. + +To Download the data required for running the tests (this data is common for +MATLAB and python tests), type: + +```bash +make tests/data/nsdcoreexampledataset.mat +``` + +Only some specific results are checked by the system tests: those can be found +in `tests/expected/matlab`. See this [README](./tests/expected/matlab/README.md) +for more information. + +To run **all** the tests and get code coverage, you can + +1. type the following in a terminal + +``` +make test-matlab +``` + +1. run `moxunit_runtests` in MATLAB to run all `test_*.m` files in in the + present working directory. + +1. run the `run_tests.m` in MATLAB + +You can also run all the tests contained in a specific `test_*.m` file directly, +by running that file only. + +#### Adding new tests + +A typical MoxUnit test file starts with with `test_` and would look something +like this. + +```matlab +function test_suite=test_sum_of_squares + + try % assignment of 'localfunctions' is necessary in Matlab >= 2016 + test_functions=localfunctions(); + catch % no problem; early Matlab versions can use initTestSuite fine + end + initTestSuite(); + +end + +function test_sum_of_squares_basic + + % given + a = 2; + b = 3; + + % when + result = sum_of_squares([a, b]) + + % then + expected = 13; + assertEqual(result, expected); + +end + +% New tests can added as new sub functions + +``` ### Continuous integration +We use Github to run several workflows for continuous integration. + +#### Tests + +The matlab tests are run by the workflow: +`.github/workflows/run_tests_matlab.yaml`. It sets up MATLAB, Moxunit and Mocov +and then then calls `.github/workflows/run_tests_ci.m` to run the tests via +`run_tests.m`. + +Those tests should be run with every push on the `master` branch and on pull +request that target the `master` branch. + +#### Demos + +The demos in the `matlab/examples` folder are run automatically in Github CI at +regular intervals. + +The matlab demos are run by the workflow: +`.github/workflows/run_demos_matlab.yaml`. The demos are run by calling +`.github/workflows/run_demos_ci.m` and also each demo is run via a MoxUnit test +(see `.github/workflows/test_demos.m`) to make sure that if the first one +crashes, then the second one will still be run (easier than setting up parallel +jobs in CI). + ## Python ### Style guide @@ -53,4 +243,4 @@ Information for anyone who would like to contribute to this repository. #### Demos -### Continuous integration \ No newline at end of file +### Continuous integration diff --git a/Makefile b/Makefile index 75e39ce..2e3a9ac 100644 --- a/Makefile +++ b/Makefile @@ -47,11 +47,11 @@ install_dev: ## install for both matlab and python developpers pip install -e . pip install -r requirements_dev.txt -lint: lint/black lint/flake8 lint/miss_hit ## check style +lint: lint/black lint/flake8 lint/miss_hit ## check style for MATLAB and python -test: test-matlab test-python +test: test-matlab test-python # run tests with MATLAB and python -tests/data/nsdcoreexampledataset.mat: +tests/data/nsdcoreexampledataset.mat: ## install test data mkdir tests/data curl -fsSL --retry 5 -o "tests/data/nsdcoreexampledataset.mat" https://osf.io/k89b2/download @@ -64,8 +64,9 @@ tests/data/nsdcoreexampledataset.mat: lint/miss_hit: ## lint and checks matlab code mh_style --fix tests && mh_metric --ci tests && mh_lint tests + mh_style --fix .github/workflows && mh_metric --ci .github/workflows && mh_lint .github/workflows -test-matlab: tests/data/nsdcoreexampledataset.mat +test-matlab: tests/data/nsdcoreexampledataset.mat ## run tests with MATLAB $(MATLAB) $(MATLAB_ARG) -r "run_tests; exit()" coverage-matlab: test-matlab diff --git a/README.md b/README.md index 336ed01..e6ba459 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +[![MATLAB test](https://github.com/cvnlab/GLMsingle/actions/workflows/run_tests_matlab.yml/badge.svg)](https://github.com/cvnlab/GLMsingle/actions/workflows/run_tests_matlab.yml) +[![MATLAB demos](https://github.com/cvnlab/GLMsingle/actions/workflows/run_demos_matlab.yml/badge.svg)](https://github.com/cvnlab/GLMsingle/actions/workflows/run_demos_matlab.yml) + # GLMsingle ![image](https://user-images.githubusercontent.com/35503086/151108958-24479034-c7f7-4734-b903-9046ba6a78ac.png) diff --git a/run_tests.m b/run_tests.m new file mode 100644 index 0000000..9ee5f41 --- /dev/null +++ b/run_tests.m @@ -0,0 +1,29 @@ +function run_tests() + + % run tests with code coverage + + tic; + + cd(fileparts(mfilename('fullpath'))); + + fprintf('\nHome is %s\n', getenv('HOME')); + + folder_to_cover = fullfile(pwd, 'matlab'); + + test_folder = fullfile(pwd, 'tests'); + + success = moxunit_runtests(test_folder, ... + '-verbose', '-recursive', '-with_coverage', ... + '-cover', folder_to_cover, ... + '-cover_xml_file', 'coverage.xml', ... + '-cover_html_dir', fullfile(pwd, 'coverage_html')); + + if success + system('echo 0 > test_report.log'); + else + system('echo 1 > test_report.log'); + end + + toc; + +end diff --git a/tests/expected/matlab/README.md b/tests/expected/matlab/README.md new file mode 100644 index 0000000..158f6a2 --- /dev/null +++ b/tests/expected/matlab/README.md @@ -0,0 +1,28 @@ +# Content + +Data used for the expected results of the system tests. + +```bash + ├── TYPEB_FITHRF.mat + ├── TYPEC_FITHRF_GLMDENOISE.mat + └── TYPED_FITHRF_GLMDENOISE_RR.mat +``` + +Contain the expected value for: + +- `HRFindex` values for `results{2:4}` for `tests/test_glmsingleunit.m`. + +`TYPED_FITHRF_GLMDENOISE_RR.mat` also contains `R2` values. + +Generated with MATLAB 2017a on Ubuntu 18.04. + +If new expected data needs to be generated, this can be done from the output of +`tests/test_GLMestimatesingletrial.m` that will be saved in `tests/outputs`. + +Some basic sanity checks can be run on those data with +`tests/expected/matlab/sanity_checks.m` + +- like making sure that the there is nothing "strange" in the range of R2 values + tested. + +![histogram_R2](./histogram_R2.png) diff --git a/tests/expected/matlab/TYPEB_FITHRF.mat b/tests/expected/matlab/TYPEB_FITHRF.mat new file mode 100644 index 0000000..2713bea Binary files /dev/null and b/tests/expected/matlab/TYPEB_FITHRF.mat differ diff --git a/tests/expected/matlab/TYPEC_FITHRF_GLMDENOISE.mat b/tests/expected/matlab/TYPEC_FITHRF_GLMDENOISE.mat new file mode 100644 index 0000000..07ac64a Binary files /dev/null and b/tests/expected/matlab/TYPEC_FITHRF_GLMDENOISE.mat differ diff --git a/tests/expected/matlab/TYPED_FITHRF_GLMDENOISE_RR.mat b/tests/expected/matlab/TYPED_FITHRF_GLMDENOISE_RR.mat new file mode 100644 index 0000000..8fe2298 Binary files /dev/null and b/tests/expected/matlab/TYPED_FITHRF_GLMDENOISE_RR.mat differ diff --git a/tests/expected/matlab/histogram_R2.png b/tests/expected/matlab/histogram_R2.png new file mode 100644 index 0000000..60dbe6f Binary files /dev/null and b/tests/expected/matlab/histogram_R2.png differ diff --git a/tests/expected/matlab/sanity_checks.m b/tests/expected/matlab/sanity_checks.m new file mode 100644 index 0000000..f336865 --- /dev/null +++ b/tests/expected/matlab/sanity_checks.m @@ -0,0 +1,21 @@ +% make sure we have no NaN / Inf in our expected results +% aslo plots a couple of figures for quick visual inspection + +clear +clc +close all + +load('TYPED_FITHRF_GLMDENOISE_RR.mat') + +assert(any(isnan(R2(:))) == 0); +assert(any(isinf(R2(:))) == 0); + +figure('name', 'histogram R2') +hist(R2(:), 100); +print('histogram_R2.png', '-dpng'); + +figure('name', 'R2') +imagesc(R2, [0 100]) + +figure('name', 'HRFindex') +imagesc(HRFindex, [1 20]) \ No newline at end of file diff --git a/tests/miss_hit.cfg b/tests/miss_hit.cfg new file mode 100644 index 0000000..ca6e42d --- /dev/null +++ b/tests/miss_hit.cfg @@ -0,0 +1,22 @@ +# miss_hit configuration +# for matlab linting and static analysis +# style guide (https://florianschanda.github.io/miss_hit/style_checker.html) +# metrics limit for the code quality (https://florianschanda.github.io/miss_hit/metrics.html) + +project_root + +octave: false + +line_length: 100 + +suppress_rule: "copyright_notice" + +tab_width: 2 + +# snake_case +regex_function_name: "[a-zA-Z]+(_[a-zA-Z]*)*" + +metric "cnest": limit 4 +metric "file_length": limit 500 +metric "cyc": limit 10 +metric "parameters": limit 5 diff --git a/tests/test_GLMestimatesingletrial.m b/tests/test_GLMestimatesingletrial.m new file mode 100644 index 0000000..c42120b --- /dev/null +++ b/tests/test_GLMestimatesingletrial.m @@ -0,0 +1,71 @@ +function test_suite = test_GLMestimatesingletrial %#ok<*STOUT> + + try % assignment of 'localfunctions' is necessary in Matlab >= 2016 + test_functions = localfunctions(); %#ok<*NASGU> + catch % no problem; early Matlab versions can use initTestSuite fine + end + + initTestSuite; + +end + +function test_GLMestimatesingletrial_system() + + % "end-to-end" test of GLMestimatesingletrial + % only checks the HRF index of 400 voxels + + [data, expected, output_dir] = set_up_test(); + + % GIVEN + design = data.design(1:3); + stimdur = data.stimdur; + tr = data.tr; + data = cellfun(@(x) x(51:70, 8:27, 1, :), data.data(1:3), 'UniformOutput', 0); + + % WHEN + results = GLMestimatesingletrial(design, ... + data, ... + stimdur, ... + tr, ... + output_dir, ... + struct('wantmemoryoutputs', [1 1 1 1])); + + % THEN + assertEqual(results{2}.HRFindex, expected{2}.HRFindex); + assertEqual(results{3}.HRFindex, expected{3}.HRFindex); + assertEqual(results{4}.HRFindex, expected{4}.HRFindex); + + assertElementsAlmostEqual(results{4}.R2, expected{4}.R2, 'absolute', 1e-1); + + clean_up(); + +end + +function [data, expected, output_dir] = set_up_test() + + test_dir = fileparts(mfilename('fullpath')); + + data_dir = fullfile(test_dir, 'data'); + data_file = fullfile(data_dir, 'nsdcoreexampledataset.mat'); + data = load(data_file); + + expected_dir = fullfile(test_dir, 'expected', 'matlab'); + load(fullfile(expected_dir, 'TYPEB_FITHRF.mat')); + expected{2}.HRFindex = HRFindex; + load(fullfile(expected_dir, 'TYPEC_FITHRF_GLMDENOISE.mat')); + expected{3}.HRFindex = HRFindex; + load(fullfile(expected_dir, 'TYPED_FITHRF_GLMDENOISE_RR.mat')); + expected{4}.HRFindex = HRFindex; + expected{4}.R2 = R2; + + output_dir = fullfile(test_dir, 'outputs', 'matlab'); + + run(fullfile(test_dir, '..', 'setup.m')); + +end + +function clean_up() + + % ununsed for now + +end