Skip to content

Commit 2a0f1c9

Browse files
priyakasimbegpriyakasimbeg
committed
switch to pyproject toml
2 parents 1ce6dea + def1a43 commit 2a0f1c9

File tree

9 files changed

+358
-313
lines changed

9 files changed

+358
-313
lines changed

‎.github/workflows/CI.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ jobs:
199199
pip install .[pytorch_cpu]
200200
- name: Run pytest tests
201201
run: |
202-
pytest -vx tests/version_test.py
202+
pytest -vx tests/test_version.py
203203
pytest -vx tests/test_num_params.py
204204
pytest -vx tests/test_param_shapes.py
205205
pytest -vx tests/test_param_types.py

‎.github/workflows/linting.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ jobs:
5050
- name: Install yapf
5151
run: |
5252
python -m pip install --upgrade pip
53-
pip install yapf==0.32
53+
pip install yapf==0.32 toml
5454
- name: Run yapf
5555
run: |
5656
yapf . --diff --recursive

‎.gitignore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,6 @@ wandb/
2323
scoring/plots/
2424

2525
!scoring/test_data/experiment_dir/study_0/mnist_jax/trial_0/eval_measurements.csv
26-
!scoring/test_data/experiment_dir/study_0/mnist_jax/trial_1/eval_measurements.csv
26+
!scoring/test_data/experiment_dir/study_0/mnist_jax/trial_1/eval_measurements.csv
27+
28+
algorithmic_efficiency/_version.py

‎CONTRIBUTING.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
- [Style Testing](#style-testing)
2323
- [Unit and Integration Tests](#unit-and-integration-tests)
2424
- [Regression Tests](#regression-tests)
25+
- [Versioning](#versioning)
2526

2627
## Contributing to MLCommons
2728

@@ -276,3 +277,15 @@ To run a regression test:
276277
2. Turn on the self-hosted runner.
277278
3. Run the self-hosted runner application for the runner to accept jobs.
278279
4. Open a pull request into mian to trigger the workflow.
280+
281+
### Versioning
282+
283+
The package version is automatically determined by the `setuptools_scm` package based on the last git tag.
284+
It follows the structure `major.minor.patch` + `devN` where `N` is the number of commits since the last tag.
285+
It automatically increments the patch version (i.e. it guesses the next version) if there are commits after the last tag.
286+
Additionally, if there are uncommitted changes, the version will include a suffix separated by a `+` character and includes the last commit hash plus the date on dirt workdir (see [setuptools_scm's documentation](https://setuptools-scm.readthedocs.io/en/latest/extending/#setuptools_scmlocal_scheme) with the default version and local scheme).
287+
You can check what version `setuptools_scm` is creating by running `python -m setuptools_scm`.
288+
289+
To create a new version, create a new release (and tag) in the GitHub UI.
290+
The package version is automatically updated to the new version.
291+
Once the package is installed, the version can be accessed as the package attribute `algorithmic_efficiency.__version__`, i.e. via `python -c "import algorithmic_efficiency; print(algorithmic_efficiency.__version__)"`.

‎algorithmic_efficiency/__init__.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
"""Algorithmic Efficiency."""
22

3-
__version__ = '0.1.0'
3+
from ._version import version as __version__
4+
5+
__all__ = ["__version__"]

‎pyproject.toml‎

Lines changed: 332 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,332 @@
1+
###############################################################################
2+
# MLCommons Algorithmic Efficiency. #
3+
###############################################################################
4+
5+
[project]
6+
name = "algorithmic_efficiency"
7+
dynamic = ["version"]
8+
description = "Codebase for the AlgoPerf: Training Algorithms benchmark"
9+
authors = [
10+
{ name = "MLCommons Algorithms Working Group", email = "[email protected]" },
11+
]
12+
license = { text = "Apache 2.0" }
13+
readme = "README.md"
14+
requires-python = ">=3.11"
15+
keywords = [
16+
"algoperf",
17+
"algorithmic-efficiency",
18+
"machine-learning",
19+
"deep-learning",
20+
"optimization",
21+
"benchmarking",
22+
"training-methods",
23+
]
24+
classifiers = [
25+
"Development Status :: 3 - Alpha",
26+
"Intended Audience :: Developers",
27+
"Intended Audience :: Science/Research",
28+
"License :: OSI Approved :: Apache Software License",
29+
"Operating System :: OS Independent",
30+
"Programming Language :: Python :: 3.8",
31+
"Programming Language :: Python :: 3.9",
32+
"Programming Language :: Python :: 3.10",
33+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
34+
]
35+
dependencies = [
36+
"absl-py==2.1.0",
37+
"networkx==3.2.1",
38+
"docker==7.1.0",
39+
"numpy>=2.0.2",
40+
"pandas>=2.0.1",
41+
"tensorflow==2.18.0",
42+
"tensorflow-datasets==4.9.7",
43+
"tensorflow-probability==0.20.0",
44+
"tensorflow-addons==0.20.0",
45+
"gputil==1.4.0",
46+
"psutil==6.1.0",
47+
"clu==0.0.12",
48+
"matplotlib>=3.9.2",
49+
"tabulate==0.9.0",
50+
]
51+
52+
[build-system]
53+
requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]
54+
build-backend = "setuptools.build_meta"
55+
56+
[tool.setuptools]
57+
py-modules = ["submission_runner"]
58+
include-package-data = true
59+
zip-safe = false
60+
61+
[tool.setuptools.packages]
62+
find = {} # Scanning implicit namespaces is active by default
63+
64+
[tool.setuptools_scm]
65+
version_file = "algorithmic_efficiency/_version.py"
66+
67+
###############################################################################
68+
# (Optional) Dependencies #
69+
###############################################################################
70+
[project.optional-dependencies]
71+
# All workloads
72+
full = [
73+
"algorithmic_efficiency[criteo1tb,fastmri,ogbg,librispeech_conformer,wmt]",
74+
]
75+
# All workloads plus development dependencies
76+
full_dev = ["algorithmic_efficiency[full,dev]"]
77+
# Dependencies for developing the package
78+
dev = [
79+
"isort==5.13.0",
80+
"pylint==2.17.4",
81+
"pytest==8.3.3",
82+
"yapf==0.32.0",
83+
"pre-commit==4.0.1",
84+
]
85+
86+
# Workloads
87+
criteo1tb = ["scikit-learn==1.5.2"]
88+
fastmri = ["h5py==3.12.0", "scikit-image==0.24.0"]
89+
ogbg = ["jraph==0.0.6.dev0", "scikit-learn==1.5.2"]
90+
librispeech_conformer = [
91+
"sentencepiece==0.2.0",
92+
"tensorflow-text==2.18.0",
93+
"pydub==0.25.1",
94+
]
95+
wmt = ["sentencepiece==0.2.0", "tensorflow-text==2.18.0"]
96+
97+
# Frameworks
98+
jax_core_deps = [
99+
"flax==0.8.4",
100+
"optax==0.2.2",
101+
"chex==0.1.86",
102+
"ml_dtypes==0.4.1",
103+
"protobuf==4.25.5",
104+
]
105+
jax_cpu = [
106+
"jax==0.4.26",
107+
"jaxlib==0.4.26",
108+
"algorithmic_efficiency[jax_core_deps]",
109+
]
110+
jax_gpu = [
111+
"jax==0.4.26",
112+
"jaxlib==0.4.26",
113+
"jax-cuda12-plugin[with_cuda]==0.4.28",
114+
"jax-cuda12-pjrt==0.4.28",
115+
"algorithmic_efficiency[jax_core_deps]",
116+
]
117+
pytorch_cpu = ["torch==2.5.1", "torchvision==0.20.1"]
118+
pytorch_gpu = [
119+
"torch==2.5.0",
120+
"torchvision==0.20.1",
121+
] # Note: omit the cuda suffix and installing from the appropriate wheel will result in using locally installed CUDA.
122+
wandb = ["wandb==0.16.5"]
123+
124+
###############################################################################
125+
# Linting Configurations #
126+
###############################################################################
127+
128+
# yapf configuration
129+
[tool.yapf]
130+
based_on_style = "yapf"
131+
each_dict_entry_on_separate_line = false
132+
split_all_top_level_comma_separated_values = true
133+
[tool.yapfignore]
134+
ignore_patterns = ["algorithmic_efficiency/_version.py"]
135+
136+
# isort configuration
137+
[tool.isort]
138+
profile = "google"
139+
140+
# pylint configuration
141+
[tool.pylint.MASTER]
142+
persistent = false
143+
ignore = "get_references_web.py,get_references_web_single_group.py,_version.py"
144+
145+
[tool.pylint.REPORTS]
146+
reports = false
147+
msg-template = "{msg_id}:{line:3} {obj}: {msg} [{symbol}]"
148+
149+
[tool.pylint.MESSAGES_CONTROL]
150+
enable = "indexing-exception,old-raise-syntax"
151+
152+
[tool.pylint.BASIC]
153+
# Required attributes for module, separated by a comma
154+
#required-attributes=
155+
# Regular expression which should only match the name
156+
# of functions or classes which do not require a docstring.
157+
no-docstring-rgx = "(__.*__|main)"
158+
# Min length in lines of a function that requires a docstring.
159+
docstring-min-length = 10
160+
# Regular expression which should only match correct module names. The
161+
# leading underscore is sanctioned for private modules by Google's style
162+
# guide.
163+
#
164+
# There are exceptions to the basic rule (_?[a-z][a-z0-9_]*) to cover
165+
# requirements of Python's module system.
166+
module-rgx = "^(_?[a-z][a-z0-9_]*)|__init__$"
167+
# Regular expression which should only match correct module level names
168+
const-rgx = "^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$"
169+
# Regular expression which should only match correct class attribute
170+
class-attribute-rgx = "^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$"
171+
# Regular expression which should only match correct class names
172+
class-rgx = "^_?[A-Z][a-zA-Z0-9]*$"
173+
# Regular expression which should only match correct function names.
174+
# 'camel_case' and 'snake_case' group names are used for consistency of naming
175+
# styles across functions and methods.
176+
function-rgx = "^(?:(?P<exempt>setUp|tearDown|setUpModule|tearDownModule)|(?P<camel_case>_?[A-Z][a-zA-Z0-9]*)|(?P<snake_case>_?[a-z][a-z0-9_]*))$"
177+
# Regular expression which should only match correct method names.
178+
# 'camel_case' and 'snake_case' group names are used for consistency of naming
179+
# styles across functions and methods. 'exempt' indicates a name which is
180+
# consistent with all naming styles.
181+
method-rgx = "(?x)^(?:(?P<exempt>_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|_testDatasetSize|setUpClass|test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|(?:test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next|(?P<camel_case>_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P<snake_case>_{0,2}[a-z][a-z0-9_]*))$"
182+
# Regular expression which should only match correct instance attribute names
183+
attr-rgx = "^_{0,2}[a-z][a-z0-9_]*$"
184+
# Regular expression which should only match correct argument names
185+
argument-rgx = "^[a-z][a-z0-9_]*$"
186+
# Regular expression which should only match correct variable names
187+
variable-rgx = "^[a-z][a-z0-9_]*$"
188+
# Regular expression which should only match correct list comprehension /
189+
# generator expression variable names
190+
inlinevar-rgx = "^[a-z][a-z0-9_]*$"
191+
# Good variable names which should always be accepted, separated by a comma
192+
good-names = "main,_"
193+
# Bad variable names which should always be refused, separated by a comma
194+
bad-names = ""
195+
# List of builtins function names that should not be used, separated by a comma
196+
#bad-functions=input,apply,reduce
197+
# List of decorators that define properties, such as abc.abstractproperty.
198+
property-classes = "abc.abstractproperty"
199+
200+
[tool.pylint.typecheck]
201+
# Tells whether missing members accessed in mixin class should be ignored. A
202+
# mixin class is detected if its name ends with "mixin" (case insensitive).
203+
ignore-mixin-members = true
204+
205+
# List of decorators that create context managers from functions, such as
206+
# contextlib.contextmanager.
207+
contextmanager-decorators = [
208+
"contextlib.contextmanager",
209+
"contextlib2.contextmanager",
210+
]
211+
212+
[tool.pylint.VARIABLES]
213+
# Tells whether we should check for unused import in __init__ files.
214+
init-import = false
215+
216+
# A regular expression matching names used for dummy variables (i.e. not used).
217+
dummy-variables-rgx = "^\\*{0,2}(_$|unused_|dummy_)"
218+
219+
# List of additional names supposed to be defined in builtins.
220+
additional-builtins = []
221+
222+
[tool.pylint.CLASSES]
223+
# List of method names used to declare (i.e. assign) instance attributes.
224+
defining-attr-methods = ["__init__", "__new__", "setUp"]
225+
226+
# Valid names for the first argument to a class method.
227+
valid-classmethod-first-arg = ["cls", "class_"]
228+
229+
[tool.pylint.EXCEPTIONS]
230+
overgeneral-exceptions = [
231+
"builtins.StandardError",
232+
"builtins.Exception",
233+
"builtins.BaseException",
234+
]
235+
236+
[tool.pylint.IMPORTS]
237+
# Deprecated modules which should not be used, separated by a comma
238+
deprecated-modules = ["regsub", "TERMIOS", "Bastion", "rexec", "sets"]
239+
240+
[tool.pylint.FORMAT]
241+
# List of checkers and warnings to disable.
242+
disable = [
243+
"abstract-method",
244+
"access-member-before-definition",
245+
"arguments-differ",
246+
"assignment-from-no-return",
247+
"attribute-defined-outside-init",
248+
"bad-mcs-classmethod-argument",
249+
"bad-option-value",
250+
"c-extension-no-member",
251+
"consider-merging-isinstance",
252+
"consider-using-dict-comprehension",
253+
"consider-using-enumerate",
254+
"consider-using-in",
255+
"consider-using-set-comprehension",
256+
"consider-using-ternary",
257+
"deprecated-method",
258+
"design",
259+
"file-ignored",
260+
"fixme",
261+
"global-statement",
262+
"import-error",
263+
"inconsistent-return-statements",
264+
"invalid-unary-operand-type",
265+
"len-as-condition",
266+
"locally-disabled",
267+
"locally-enabled",
268+
"misplaced-comparison-constant",
269+
"missing-docstring",
270+
"multiple-imports",
271+
"no-else-return",
272+
"no-member",
273+
"no-name-in-module",
274+
"no-self-use",
275+
"no-value-for-parameter",
276+
"not-an-iterable",
277+
"not-context-manager",
278+
"pointless-except",
279+
"protected-access",
280+
"redefined-argument-from-local",
281+
"signature-differs",
282+
"similarities",
283+
"simplifiable-if-expression",
284+
"star-args",
285+
"super-init-not-called",
286+
"suppressed-message",
287+
"too-many-function-args",
288+
"trailing-comma-tuple",
289+
"trailing-newlines",
290+
"ungrouped-imports",
291+
"unnecessary-pass",
292+
"unsubscriptable-object",
293+
"unused-argument",
294+
"useless-object-inheritance",
295+
"useless-return",
296+
"useless-suppression",
297+
"wrong-import-order",
298+
"wrong-import-position",
299+
"unneeded-not",
300+
"unexpected-keyword-arg",
301+
"redundant-keyword-arg",
302+
"unspecified-encoding",
303+
"logging-fstring-interpolation",
304+
"consider-using-f-string",
305+
"use-dict-literal",
306+
]
307+
# Maximum number of characters on a single line.
308+
max-line-length = 80
309+
ignore-long-lines = "(?x)(^\\s*(import|from)\\s|^\\s*(\\#\\ )?<?(https?|ftp):\\/\\/[^\\s\\/$.?#].[^\\s]*>?$|^[a-zA-Z_][a-zA-Z0-9_]*\\s*=\\s*('[^']\\S+'|\"[^\"]\\S+\"))"
310+
# Maximum number of lines in a module
311+
max-module-lines = 99999
312+
# String used as indentation unit. We differ from PEP8's normal 4 spaces.
313+
indent-string = ' '
314+
single-line-if-stmt = true
315+
# Do not warn about multiple statements on a single line for constructs like
316+
# if test: stmt
317+
[tool.pylint.LOGGING]
318+
logging-modules = "logging,absl.logging"
319+
# Add logging modules.
320+
[tool.pylint.MISCELLANEOUS]
321+
# Maximum line length for lambdas
322+
#short-func-length=1
323+
# List of module members that should be marked as deprecated.
324+
# All of the string functions are listed in 4.1.4 Deprecated string functions
325+
# in the Python 2.4 docs.
326+
#deprecated-members=string.atof,string.atoi,string.atol,string.capitalize,string.expandtabs,string.find,string.rfind,string.index,string.rindex,string.count,string.lower,string.split,string.rsplit,string.splitfields,string.join,string.joinfields,string.lstrip,string.rstrip,string.strip,string.swapcase,string.translate,string.upper,string.ljust,string.rjust,string.center,string.zfill,string.replace,sys.exitfunc,sys.maxint
327+
# List of exceptions that do not need to be mentioned in the Raises section of
328+
# a docstring.
329+
#ignore-exceptions=AssertionError,NotImplementedError,StopIteration,TypeError
330+
# Number of spaces of indent required when the last token on the preceding line
331+
# is an open (, [, or {.
332+
indent-after-paren = 4

0 commit comments

Comments
 (0)