chore: improving codebase; adding semantic releases; fixing scripts & tests post migration (#2)

* fix: patching helper scripts; adding pre-commit; bumping compiler version

* ci: adding semantic releases

* chore: patching pipeline

* chore: improving cd

* chore: patching ci

* chore: refining ci

* chore: refine ci

Author: aorumbayev

.github/workflows/cd.yaml

@@ -1,15 +1,20 @@
 name: Continuous Delivery of Algorand Python Testing package
 
 on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - "**.md"
+      - "docs/**"
+      - "scripts/**"
+      - "examples/**"
+      - "tests/**"
+
   workflow_dispatch:
     inputs:
-      publish_pypi:
-        description: "Publish to PyPi?"
-        type: boolean
-        required: true
-        default: false
-      run_checks:
-        description: "Run checks?"
+      prerelease:
+        description: "Prerelease?"
         type: boolean
         required: true
         default: true
@@ -35,10 +40,12 @@ jobs:
       contents: write
       packages: read
     env:
-      DRY_RUN: ${{ inputs.dry_run && '--noop' || '' }}
+      DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.dry_run && '--noop' || '') || '' }}
+      PRERELEASE: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.prerelease && 'true' || '') || 'true' }}
     steps:
       - uses: actions/checkout@v4
         with:
+          fetch-depth: 0
           token: ${{ secrets.RELEASE_GITHUB_TOKEN }}
 
       - name: Install hatch
@@ -63,20 +70,25 @@ jobs:
         run: hatch build
 
       - name: Run tests (codebase)
-        run: hatch run test:codebase
+        run: hatch run tests
 
       - name: Run tests (examples)
         run: hatch run examples:tests
 
-      - name: Create Unit Testing Wheel
-        run: hatch build
-
       - uses: actions/upload-artifact@v4 # upload artifacts so they are retained on the job
         with:
           path: dist
 
-      - name: Publish to PyPI - Unit Testing
-        if: ${{ !inputs.dry_run && inputs.publish_pypi }}
+      - name: Python Semantic Release
+        if: ${{ github.ref == 'refs/heads/main' }}
+        uses: python-semantic-release/python-semantic-release@master
+        with:
+          github_token: ${{ secrets.RELEASE_GITHUB_TOKEN }}
+          prerelease: ${{ env.PRERELEASE == 'true' }}
+          root_options: $DRY_RUN
+
+      - name: Publish to PyPI
+        if: ${{ !inputs.dry_run }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages-dir: dist

.github/workflows/ci.yaml

@@ -33,7 +33,7 @@ jobs:
         run: hatch build
 
       - name: Run tests (codebase)
-        run: hatch run test:codebase
+        run: hatch run tests
 
       - name: Run tests (examples)
         run: hatch run examples:tests

.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+repos:
+  - repo: local
+    hooks:
+      - id: pre-commit
+        name: pre-commit
+        description: "Run pre-commit task via hatch"
+        entry: hatch run pre_commit
+        language: system
+        additional_dependencies: []
+        minimum_pre_commit_version: "0"
+
+      - id: examples-pre-commit
+        name: examples-pre-commit
+        description: "Run examples venv pre-commit task via hatch"
+        entry: hatch run examples:pre_commit
+        language: system
+        additional_dependencies: []
+        minimum_pre_commit_version: "0"

CONTRIBUTING.md

@@ -1,11 +1,14 @@
-# Development dependencies
+# Contributing to `algorand-python-testing`
+
+Welcome to the Algorand Python Testing library! This guide will help you get started with the project and contribute to its development.
 
 # Development Dependencies
 
 Our development environment relies on the following tools:
 
 -   **Python 3.12**
 -   **[Hatch](https://hatch.pypa.io/1.9/install/)**: A modern, extensible Python project manager.
+-   **[pre-commit](https://pre-commit.com/)**: A framework for managing and maintaining code quality.
 
 ## Common Commands
 
@@ -22,6 +25,18 @@ Here are some common commands you will use with Hatch:
 -   **Regenerate typed clients for example contracts:** `hatch run refresh_test_artifacts`
 -   **Coverage check of existing progress on implementing AlgoPy Stubs:** `hatch run check_stubs_cov`
 
+# Using `pre-commit`
+
+Execute `pre-commit install` to ensure auto run of hatch against `src` and `examples` folders
+
 # Examples folder
 
 Examples folder uses a dedicated 'venv.examples' virtual environment managed by Hatch that simulates a user environment with both algorand-python and algorand-python-testing installed explicitly. This is useful for testing new features or bug fixes in the testing library.
+
+-   **Pre-commit checks against examples:** `hatch run examples:pre-commit`
+
+# Release automation
+
+Project relies on [python-semantic-release](https://python-semantic-release.readthedocs.io/en/latest/) for release automation.
+
+Releases are triggered by a `workflow_dispatch` event on the `main` branch with `prerelease` set to accordingly using the `.github/workflows/cd.yaml` file.

README.md

@@ -1,4 +1,15 @@
-# Algorand Python Testing
+<div align="center">
+<a href="https://github.com/algorandfoundation/algorand-python-testing"><img src="https://bafybeiaibjaf6zy6hvef2rrysaacsfsyb3hw4qqtgn657gw7k5tdzqdxzi.ipfs.nftstorage.link/" width=60%></a>
+</div>
+
+<p align="center">
+    <a target="_blank" href="https://algorandfoundation.github.io/algorand-python-testing/"><img src="https://img.shields.io/badge/docs-repository-74dfdc?logo=github&style=flat.svg" /></a>
+    <a target="_blank" href="https://developer.algorand.org/algokit/"><img src="https://img.shields.io/badge/learn-AlgoKit-74dfdc?logo=algorand&mac=flat.svg" /></a>
+    <a target="_blank" href="https://github.com/algorandfoundation/algorand-python-testing"><img src="https://img.shields.io/github/stars/algorandfoundation/algorand-python-testing?color=74dfdc&logo=star&style=flat" /></a>
+    <a target="_blank" href="https://developer.algorand.org/algokit/"><img  src="https://api.visitorbadge.io/api/visitors?path=https%3A%2F%2Fgithub.com%2Falgorandfoundation%2Falgorand-python-testing&countColor=%2374dfdc&style=flat" /></a>
+</p>
+
+---
 
 Algorand Python Testing is a companion package to [Algorand Python](https://github.com/algorandfoundation/puya) that enables efficient unit testing of Algorand Python smart contracts in an offline environment. It emulates key AVM behaviors without requiring a network connection, offering fast and reliable testing capabilities with a familiar Pythonic interface.
 

examples/zk_whitelist/contract.py

@@ -11,6 +11,7 @@
     Txn,
     UInt64,
     itxn,
+    op,
     subroutine,
 )
 from algopy.arc4 import (
@@ -62,7 +63,9 @@ def add_address_to_whitelist(self, address: Bytes32, proof: DynamicArray[Bytes32
         # The verifier expects public inputs to be in the curve field, but an
         # Algorand address might represent a number larger than the field
         # modulus, so to be safe we take the address modulo the field modulus
-        address_mod = Bytes32.from_bytes((py.BigUInt.from_bytes(address.bytes) % curve_mod).bytes)
+        address_mod = Bytes32.from_bytes(
+            (py.BigUInt.from_bytes(address.bytes) % curve_mod).bytes or op.bzero(32)
+        )
 
         # Verify the proof by calling the deposit verifier app
         verified = verify_proof(

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "algorand-python-testing"
-version = "0.2.1"
+version = "0.0.2-beta.1"
 description = 'Algorand Python testing library'
 readme = "README.md"
 requires-python = ">=3.12"
@@ -52,7 +52,7 @@ type = "virtual"
 path = ".venv"
 python = "3.12"
 dependencies = [
-    "puyapy>=2.1",
+    "puyapy>=2.1.2",
     "pytest>=7.4",
     "pytest-mock>=3.10.0",
     "pytest-xdist[psutil]>=3.3",
@@ -81,6 +81,7 @@ check = [
 ]
 # type checks algorand-python-testing code
 mypy_testing = "mypy . --exclude examples"
+tests = "pytest --cov --cov-report xml"
 
 # isolated environments for dev tooling
 [tool.hatch.envs.lint]
@@ -109,6 +110,7 @@ fix_examples = [ 'black examples', 'ruff check --fix examples']
 path = ".venv.cicd"
 dependencies = [
     "algokit",
+    "python-semantic-release>=9.8.5",
 ]
 [tool.hatch.envs.cicd.scripts]
 localnet_start = [
@@ -158,17 +160,6 @@ check = [
     "hatch run mypy examples",
 ]
 
-[tool.hatch.envs.test]
-dependencies = [
-    "pytest>=7.4",
-    "pytest-cov>=4.1.0",
-]
-[tool.hatch.envs.test.scripts]
-codebase = "pytest --cov --cov-report xml"
-
-[tool.hatch.envs.test.env-vars]
-PYTHONPATH = "./src"
-
 # tool configurations
 [tool.black]
 line-length = 99
@@ -277,3 +268,30 @@ follow_imports = "skip"
 
 [tool.pytest.ini_options]
 addopts = "-n auto --cov-config=.coveragerc"
+pythonpath = ['src']
+
+# === Semantic releases config ===
+
+[tool.semantic_release]
+version_toml = ["pyproject.toml:project.version"]
+build_command = "hatch build"
+commit_message = "{version}\n\nAutomatically generated by python-semantic-release"
+tag_format = "v{version}"
+major_on_zero = true
+
+[tool.semantic_release.branches.main]
+match = "main"
+prerelease_token = "beta"
+prerelease = false
+
+[tool.semantic_release.commit_parser_options]
+allowed_tags = ["build", "chore", "ci", "docs", "feat", "fix", "perf", "style", "refactor", "test"]
+minor_tags = ["feat"]
+patch_tags = ["fix", "perf", "docs"]
+
+[tool.semantic_release.publish]
+dist_glob_patterns = ["dist/*.whl"] # order here is important to ensure compiler wheel is published first
+upload_to_vcs_release = true
+
+[tool.semantic_release.remote.token]
+env = "GITHUB_TOKEN"

scripts/check_stubs_cov.py

@@ -1,17 +1,16 @@
 import ast
 import inspect
+import site
 import sys
 from collections.abc import Iterable
 from pathlib import Path
 from typing import NamedTuple
 
 from prettytable import PrettyTable
 
-# TODO: update to follow the new project structure
-
-PROJECT_ROOT = (Path(__file__).parent / "..").resolve()
-VCS_ROOT = (PROJECT_ROOT / "..").resolve()
-STUBS = VCS_ROOT / "stubs" / "algopy-stubs"
+PROJECT_ROOT = Path(__file__).parent.parent.resolve()
+SITE_PACKAGES = Path(site.getsitepackages()[0])
+STUBS_ROOT = SITE_PACKAGES / "algopy-stubs"
 IMPL = PROJECT_ROOT / "src"
 ROOT_MODULE = "algopy"
 
@@ -49,13 +48,20 @@ def coverage(self) -> float:
 
 
 def main() -> None:
+    clear_algopy_content()
     stubs = collect_public_stubs()
     coverage = collect_coverage(stubs)
     print_results(coverage)
 
 
+def clear_algopy_content() -> None:
+    algopy_path = SITE_PACKAGES / "algopy.py"
+    if algopy_path.exists():
+        algopy_path.write_text("")
+
+
 def collect_public_stubs() -> dict[str, ASTNodeDefinition]:
-    stubs_root = STUBS / "__init__.pyi"
+    stubs_root = STUBS_ROOT / "__init__.pyi"
     stubs_ast = _parse_python_file(stubs_root)
     result = dict[str, ASTNodeDefinition]()
     for stmt in stubs_ast.body:
@@ -80,15 +86,15 @@ def collect_imports(
     dest = alias.asname
     # from module import *
     if src == "*" and dest is None:
-        for defn_name, defn in collect_stubs(STUBS, relative_module).items():
+        for defn_name, defn in collect_stubs(STUBS_ROOT, relative_module).items():
             yield f"{ROOT_MODULE}.{defn_name}", defn
     # from root import src as src
     elif stmt.module == ROOT_MODULE and dest == src:
-        for defn_name, defn in collect_stubs(STUBS, src).items():
+        for defn_name, defn in collect_stubs(STUBS_ROOT, src).items():
             yield f"{ROOT_MODULE}.{dest}.{defn_name}", defn
     # from foo.bar import src as src
     elif dest == src:
-        stubs = collect_stubs(STUBS, relative_module)
+        stubs = collect_stubs(STUBS_ROOT, relative_module)
         yield f"{ROOT_MODULE}.{src}", stubs[src]
     else:
         raise NotImplementedError
@@ -138,7 +144,7 @@ def collect_coverage(stubs: dict[str, ASTNodeDefinition]) -> list[CoverageResult
         result.append(
             CoverageResult(
                 full_name=full_name,
-                stub_file=str(stub.path.relative_to(STUBS)),
+                stub_file=str(stub.path.relative_to(STUBS_ROOT)),
                 impl_file=impl_file,
                 coverage=coverage.coverage if coverage else 0,
                 missing=", ".join(coverage.missing if coverage else []),
@@ -185,15 +191,25 @@ def _parse_python_file(filepath: Path) -> ast.Module:
 
 def _get_impl_coverage(symbol: str, stub: ASTNodeDefinition) -> ImplCoverage | None:
     import importlib
+    import sys
+    from pathlib import Path
+
+    # Add the src directory to the Python path
+    src_path = Path(__file__).parent.parent / "src"
+    sys.path.insert(0, str(src_path))
 
     module, name = symbol.rsplit(".", maxsplit=1)
     try:
+        # Use importlib.import_module for both algopy and non-algopy modules
         mod = importlib.import_module(module)
-    except ImportError:
+    except ImportError as ex:
+        print(f"Error importing {module}: {ex}")
         return None
+
     try:
         impl = getattr(mod, name)
     except AttributeError:
+        print(f"Attribute {name} not found in module {module}")
         return None
 
     try:

scripts/refresh_test_artifacts.py

@@ -27,6 +27,8 @@ def compile_contract(folder: Path) -> None:
         "hatch",
         "run",
         "puyapy",
+        "--log-level",
+        "debug",
         str(contract_path),
         "--out-dir",
         "data",