Skip to content

Fix ml_base dependency #37721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Abacn merged 2 commits into from
Feb 27, 2026
+24 −10
Merged

Fix ml_base dependency #37721

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
39394c2
Fix ml_base dependency
Abacn Feb 26, 2026
503189f
install twice
Abacn Feb 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3165,14 +3165,16 @@ class BeamModulePlugin implements Plugin<Project> {
def distTarBall = "${pythonRootDir}/build/apache-beam.tar.gz"
def packages = "gcp,test,aws,azure,dataframe"
def extra = project.findProperty('beamPythonExtra')
if (extra) {
packages += ",${extra}"
}

project.exec {
executable 'sh'
args '-c', ". ${project.ext.envdir}/bin/activate && pip install --pre --retries 10 ${distTarBall}[${packages}]"
}
if (extra) {
project.exec {
executable 'sh'
args '-c', ". ${project.ext.envdir}/bin/activate && pip install --pre --retries 10 ${distTarBall}[${extra}]"
}
}
}
}

Expand Down
24 changes: 18 additions & 6 deletions sdks/python/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,13 +163,15 @@ def cythonize(*args, **kwargs):
milvus_dependency = ['pymilvus>=2.5.10,<3.0.0']

ml_base = [
'embeddings',
'embeddings>=0.0.4', # 0.0.3 crashes setuptools
'onnxruntime',
'langchain',
'sentence-transformers>=2.2.2',
'skl2onnx',
'pyod',
'pyod>=0.7.6', # 0.7.5 crashes setuptools
'tensorflow',
# tensorflow transient dep, lower versions not compatible with Python3.10+
'absl-py>=0.12.0',
'tensorflow-hub',
'tf2onnx',
'torch',
Expand Down Expand Up @@ -547,10 +549,12 @@ def get_portability_package_data():
# tests due to tag check introduced since pip 24.2
# https://github.com/apache/beam/issues/31285
# 'xgboost<2.0', # https://github.com/apache/beam/issues/31252
] + ml_base + milvus_dependency,
] + ml_base,
'p312_ml_test': [
'datatable',
] + ml_base,
# maintainer: milvus tests only run with this extension. Make sure it
# is covered by docker-in-docker test when changing py version
'p313_ml_test': ml_base + milvus_dependency,
'aws': ['boto3>=1.9,<2'],
'azure': [
Expand Down Expand Up @@ -584,7 +588,11 @@ def get_portability_package_data():
# For more info, see
# https://docs.google.com/document/d/1c84Gc-cZRCfrU8f7kWGsNR2o8oSRjCM-dGHO9KvPWPw/edit?usp=sharing
'torch': ['torch>=1.9.0,<2.8.0'],
'tensorflow': ['tensorflow>=2.12rc1,<2.21'],
'tensorflow': [
'tensorflow>=2.12rc1,<2.21',
# tensorflow transitive dep
'absl-py>=0.12.0'
],
'transformers': [
'transformers>=4.28.0,<4.56.0',
'tensorflow>=2.12.0',
Expand All @@ -593,7 +601,9 @@ def get_portability_package_data():
'ml_cpu': [
'tensorflow>=2.12.0',
'torch==2.8.0+cpu',
'transformers>=4.28.0,<4.56.0'
'transformers>=4.28.0,<4.56.0',
# tensorflow transient dep
'absl-py>=0.12.0'
],
'redis': ['redis>=5.0.0,<6'],
'tft': [
Expand All @@ -610,7 +620,9 @@ def get_portability_package_data():
'tensorflow==2.11.0',
'tf2onnx==1.13.0',
'skl2onnx==1.13',
'transformers==4.25.1'
'transformers==4.25.1',
# tensorflow transient dep
'absl-py>=0.12.0'
],
'xgboost': ['xgboost>=1.6.0,<2.1.3', 'datatable==1.0.0'],
'tensorflow-hub': ['tensorflow-hub>=0.14.0,<0.16.0'],
Expand Down
Loading