setup.py

"""Setup script for NexForge Tokenizer Builder."""

from setuptools import setup, find_packages

with open("README.md", "r", encoding="utf-8") as fh:
    long_description = fh.read()

setup(
    name="nexforgetokenizer",
    version="0.1.0",
    author="NexForge Team",
    description="High-performance tool for creating Python code tokenizers with adaptive resource management",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/nexforge/nexforgetokenizer",
    package_dir={"": "src"},
    packages=find_packages(where="src"),
    python_requires=">=3.8",
    install_requires=[
        "torch>=1.9.0",
        "tokenizers>=0.12.0",
        "tqdm>=4.62.0",
        "psutil>=5.9.0",
        "numpy>=1.20.0",  # Optional but recommended for improved performance
    ],
    extras_require={
        "dev": [
            "pytest>=6.0",
            "black>=21.7b0",
            "isort>=5.0.0",
            "mypy>=0.910",
            "pylint>=2.11.0",
        ],
    },
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: Other/Proprietary License",
        "Operating System :: OS Independent",
        "Intended Audience :: Developers",
        "Topic :: Software Development :: Libraries :: Python Modules",
        "Topic :: Text Processing :: Linguistic",
    ],
)