diff --git a/.github/workflows/python-test.yaml b/.github/workflows/python-test.yaml index 9c80bd2..1f9c539 100644 --- a/.github/workflows/python-test.yaml +++ b/.github/workflows/python-test.yaml @@ -3,6 +3,10 @@ name: Upload Python Package on: push: branches: [develop] + pull_request: + branches: + - master + - develop permissions: contents: read @@ -13,14 +17,19 @@ jobs: run: working-directory: opustools_pkg - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - name: Set up Python + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: - python-version: '3.x' + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/opustools_pkg/opustools/opus_get.py b/opustools_pkg/opustools/opus_get.py index b259a06..6b3b8d6 100644 --- a/opustools_pkg/opustools/opus_get.py +++ b/opustools_pkg/opustools/opus_get.py @@ -49,7 +49,7 @@ def __init__(self, source=None, target=None, directory=None, self.fromto = [source, target] self.fromto.sort() - self.url = 'http://opus.nlpl.eu/opusapi/?' + self.url = 'https://opus.nlpl.eu/opusapi' #self.url = 'http://127.0.0.1:5000/?' urlparts = [(source, 'source'), (target, 'target'), @@ -57,13 +57,15 @@ def __init__(self, source=None, target=None, directory=None, (preprocess, 'preprocessing')] self.parameters = {} + url_parameters = '' for a in urlparts: if a[0]: self.parameters[a[1]] = a[0] if a[0] == ' ': - self.url += a[1] + '=&' + url_parameters += a[1] + '=&' else: - self.url += a[1] + '=' + a[0] + '&' + url_parameters += a[1] + '=' + a[0] + '&' + self.url += '?' + url_parameters if not os.path.exists(download_dir): os.makedirs(download_dir) @@ -163,7 +165,7 @@ def download(self, corpora, total_size): reporthook=self.progress_status) print('') except urllib.error.URLError as e: - print('Unable to retrieve the data.') + print(f'Unable to retrieve the data from {self.url}: {e}') return def get_file_info_output(self, c): @@ -193,24 +195,23 @@ def get_files(self): if self.local_db: languages = self.dbo.run_languages_query(self.parameters) else: - languages = self.get_response(self.url+'languages=True')['languages'] + languages = self.get_response(self.url + '?languages=True')['languages'] print(', '.join([str(l) for l in languages])) return elif self.list_corpora: if self.local_db: corpus_list = self.dbo.run_corpora_query(self.parameters) else: - corpus_list = self.get_response(self.url+'corpora=True')['corpora'] + corpus_list = self.get_response(self.url + '?corpora=True')['corpora'] print(', '.join([str(c) for c in corpus_list])) return else: corpora, total_size = self.get_corpora_data() except urllib.error.URLError as e: - print('Unable to retrieve the data.') + print(f'Unable to retrieve the data from {self.url}: {e}') return if not self.list_resources: self.download(corpora, total_size) else: self.print_files(corpora, total_size) - diff --git a/opustools_pkg/tests/test_opus_get.py b/opustools_pkg/tests/test_opus_get.py index d627ab8..72c5657 100644 --- a/opustools_pkg/tests/test_opus_get.py +++ b/opustools_pkg/tests/test_opus_get.py @@ -40,7 +40,7 @@ def test_get_files_invalid_url(self): opg.get_files() sys.stdout = old_stdout - self.assertEqual(printout.getvalue(), 'Unable to retrieve the data.\n') + self.assertIn('Unable to retrieve the data', printout.getvalue()) @mock.patch('opustools.opus_get.input', create=True) def test_download_invalid_url(self, mocked_input): @@ -55,7 +55,7 @@ def test_download_invalid_url(self, mocked_input): opg.download(corpora, total_size) sys.stdout = old_stdout - self.assertEqual(printout.getvalue(), 'Unable to retrieve the data.\n') + self.assertIn('Unable to retrieve the data', printout.getvalue()) @mock.patch('opustools.opus_get.input', create=True) def test_dont_list_files_that_are_already_in_path(self, mocked_input): @@ -136,4 +136,3 @@ def test_indonesian(self): if __name__ == '__main__': unittest.main() -