From b3a762f8c6eb1e5b396991407d7f56241158711d Mon Sep 17 00:00:00 2001 From: Rafael Greca Date: Tue, 3 Dec 2024 17:38:10 -0300 Subject: [PATCH 1/4] improving readme and documentations --- README.md | 259 +++++++++++++++++++++++++--------------- docs/AUTHORS.md | 15 +-- docs/CODE_OF_CONDUCT.md | 105 +++++++--------- docs/CONTRIBUTING.md | 24 +++- docs/ROADMAP.md | 106 ++++++++++++++++ 5 files changed, 334 insertions(+), 175 deletions(-) create mode 100644 docs/ROADMAP.md diff --git a/README.md b/README.md index db6f4ec..5fec82a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ + ## Badges @@ -15,17 +16,45 @@ - +

+ +
+ Table of Contents +
    +
  1. About The Project
    2. +
  2. Installation
    3. +
  3. Features
    4. +
  4. Examples
    5. +
  5. Running Tests
    6. +
  6. Roadmap
    7. +
  7. Contributing
    8. +
  8. License
    9. +
  9. Authors
    10. +
  10. Acknowledgments
    11. +
+
+ # ScratchML -A Python library called ScratchML was created to build the most fundamental Machine Learning models from scratch (using only Numpy), emphasizing producing user-friendly, straightforward, easy-to-use, well-organized implementations for novices and enthusiasts. +With the goal of creating user-friendly, simple, and easy-to-use implementations for both beginners and enthusiasts, a Python library called ScratchML was developed to build the most basic Machine Learning (ML) models from scratch (using only Numpy). + +What sets this library apart from other GitHub codes and libraries that also implement ML models from scratch is: + +* well-organized implementations that make use of Object-oriented Programming (OOP) concepts +* straightforward and user-friendly implementations that help novices fully comprehend the concept behind the fundamental algorithms +* Continuous Integration (CI) that includes unit testing cases to ensure that the implementations are functioning as intended +* comparison of the variables and results, to the extent possible, with Scikit-Learn's implementation +* implementations for both binary and multiclass classifications +* does not cover only ML algorithms but also known metrics for classification and regression tasks, the most-used preprocessing steps (such as applying encoders, scalers, and splitting the data into different sets), and more! + +Disclaimer: The goal of this library is to provide code that is simpler, easier to understand, and more approachable for artificial intelligence enthusiasts and beginners who want to contribute to an open-source repository or who want to learn more about how algorithms operate. It is not meant to replace existing libraries that are better, more optimized, and have a wider variety of implemented algorithms (such as scikit-learn, PyTorch, Keras, and Tensorflow). -Disclaimer: This library is not intended to surpass those that already exist and which are better, more optimized, and with more diversity of implemented algorithms (such as scikit-learn, PyTorch, Keras, and Tensorflow), but rather to provide code that is easier to understand, simple, and friendly for beginners and enthusiasts in the field of artificial intelligence who wish to gain a deeper understanding of how algorithms work or who want to contribute to an open-source repository. +

(back to top)

## Installation @@ -35,6 +64,18 @@ To install this package, first clone the repository to the directory of your cho git clone https://github.com/rafaelgreca/scratchml.git ``` +Use the following command to install the pre-commit package manager: + +```bash +pip install pre-commit +``` + +Activate pre-commit using the following command: + +```bash +pre-commit install +``` + ### Using Virtual Environment Create a virtual environment (ideally using conda) and install the requirements with the following command: @@ -45,7 +86,7 @@ conda activate scratchml pip install -r requirements/requirements.txt ``` -### Using Docker +### (RECOMMENDED) Using Docker Build the Docker image using the following command: @@ -59,10 +100,94 @@ Run the Docker container using the following command: sudo docker run -d -p 8000:5000 --name scratchml scratchml ``` -## Usage/Examples +

(back to top)

+ +## Features + +Activation functions: + +- ELU (added in version 8.1.0) +- Leaky ReLU (added in version 8.1.0) +- Linear (added in version 8.1.0) +- ReLU (added in version 8.1.0) +- SELU (added in version 8.1.0) +- Sigmoid (added in version 8.1.0) +- Softmax (added in version 8.1.0) +- SoftPlus (added in version 8.1.0) +- TanH (added in version 8.1.0) + +Algorithms: + +- Decision Tree Classifier (added in version 5.0.0) and Decision Tree Regressor (added in version 6.0.0) +- KMeans (added in version 4.0.0) +- KNN Classifier and KNN Regressor (added in version 2.0.0) +- Linear Regression (added in version 1.0.0) +- Logistic Regression (added in version 1.0.0) +- MLP Classifier (added in version 9.0.0) and MLP Regressor (added in version 10.0.0) +- Guassian Naive Bayes (added in version 3.0.0) +- Perceptron (added in version 4.0.0) +- PCA (added in version 8.0.0) +- Random Forest Classifier and Random Forest Regressor (added in version 7.0.0) +- Support Vector Classifier and Support Vector Regressor (added in version 10.0.0) + +Data split functions: + +- KFold (added in version 1.0.0) +- Split into Batches (added in version 9.0.0) +- Stratify KFold (added in version 1.0.0) +- Train Test Split (added in version 1.0.0) + +Distance metrics: + +- Chebyshev (added in version 2.0.0) +- Euclidean (added in version 2.0.0) +- Manhattan (added in version 2.0.0) +- Minkowski (added in version 2.0.0) + +Encoders: + +- Label Encoding (added in version 1.0.0) +- One-hot Encoding (added in version 1.0.0) + +Loss functions: + +- Binary Cross Entropy (added in version 9.0.0) +- Cross Entropy (added in version 9.0.0) + +Metrics: -See the `examples` folder to see some use cases. +- Accuracy (added in version 1.0.0) +- Confusion Matrix (added in version 1.0.0) +- F1 Score (added in version 1.0.0) +- False Positive Rate (added in version 1.0.0) +- Max Error (added in version 1.0.0) +- Mean Absolute Error (added in version 1.0.0) +- Mean Absolute Percentage Error (added in version 1.0.0) +- Mean Squared Error (added in version 1.0.0) +- Mean Squared Logarithmic Error (added in version 1.0.0) +- Median Absolute Error (added in version 1.0.0) +- Precision (added in version 1.0.0) +- R Squared (added in version 1.0.0) +- Recall (added in version 1.0.0) +- ROC AUC Score (added in version 2.1.0) +- Root Mean Squared Error (added in version 1.0.0) +- True Positive Rate (added in version 1.0.0) +Regularization functions: + +- L1 (added in version 1.0.0) +- L2 (added in version 1.0.0) + +Scalers: + +- Standard Scaler (added in version 1.0.0) +- Min Max Scaler (added in version 1.0.0) + +## Examples + +Check the `examples` folder to see some examples of how to use each functionality of this library. + +

(back to top)

## Running Tests @@ -74,7 +199,7 @@ Run the following command on the root folder: python3 -m unittest discover -p 'test_*.py' ``` -### Using Docker +### (RECOMMENDED) Using Docker Build the Docker image using the following command: @@ -88,110 +213,48 @@ Run the Docker container using the following command: sudo docker run -d -p 8001:5000 --name test_scratchml test_scratchml ``` -## Roadmap +

(back to top)

-Implementations: - -- [x] Scalers - - [x] [StandardScaler](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/scalers.py#L155) - - [x] [MinMaxScaler](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/scalers.py#L37) -- [ ] Regularizations - - [x] [L1](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/regularizations.py#L4) - - [x] [L2](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/regularizations.py#L27) - - [ ] Batch Normalization -- [x] Activation functions - - [x] [Sigmoid](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L109) - - [x] [ReLU](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L23) - - [x] [Linear](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L4) - - [x] [Softmax](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L132) - - [x] [TanH](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L84) - - [x] [ELU](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L42) - - [x] [Leaky ReLU](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L65) - - [x] [SoftPlus](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L157) - - [x] [SELU](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L180) -- [x] Loss functions - - [x] [Binary Cross Entropy](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/losses.py#L4) - - [x] [Cross Entropy](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/losses.py#L33) -- [x] Metrics - - [x] Regression Metrics - - [x] [Mean Squared Error (MSE)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L7) - - [x] [Root Mean Squared Error (RMSE)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L29) - - [x] [Mean Absolute Error (MAE)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L51) - - [x] [Median Absolute Error (MedAE)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L73) - - [x] [Mean Absolute Percentage Error (MAPE)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L95) - - [x] [Mean Squared Logarithmic Error (MSLE)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L128) - - [x] [Max Error (ME)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L156) - - [x] [R Squared (R2)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L180) - - [x] Classification Metrics - - [x] [Accuracy](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L200) - - [x] [Precision](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L215) - - [x] [Recall](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L272) - - [x] [F1-Score](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L329) - - [x] [Confusion Matrix](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L373) - - [x] [ROC AUC Score](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L474) - - [x] [False Positive Rate (FPR)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L458) - - [x] [True Positive Rate (TPR)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L442) -- [x] Distances - - [x] [Euclidean](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/distances.py#L6) - - [x] [Manhattan](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/distances.py#L26) - - [x] [Chebyshev](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/distances.py#L46) - - [x] [Minkowski](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/distances.py#L66) -- [x] Encoders - - [x] [One-hot Encoding](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/encoders.py#L133) - - [x] [Label Encoding](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/encoders.py#L39) -- [x] Splitters - - [x] [KFold](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/utils.py#L42) - - [x] [Stratify KFold](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/utils.py#L42) - - [x] [Train Test Split](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/utils.py#L187) - - [x] [Split Into Batches](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/utils.py#L5) -- [ ] Models - - [x] [Linear Regression](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/linear_regression.py) - - [x] [Logistic Regression](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/logistic_regression.py) - - [x] SVM - - [x] [SVC](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/svc.py) - - [x] [SVR](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/svr.py) - - [x] KNN - - [x] [KNN Classifier](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/knn.py#L236) - - [x] [KNN Regressor](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/knn.py#L375) - - [x] [Naive Bayes](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/naive_bayes.py) - - [x] Random Forest - - [x] [Random Forest Classifier](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/random_forest.py#L291) - - [x] [Random Forest Regressor](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/random_forest.py#L445) - - [x] Decision Tree - - [x] [Decision Tree Classifier](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/decision_tree.py#L525) - - [x] [Decision Tree Regressor](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/decision_tree.py#L640) - - [x] [Perceptron](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/perceptron.py) - - [x] MLP - - [x] [MLP Classifier](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/multilayer_perceptron.py#L569) - - [x] [MLP Regressor](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/multilayer_perceptron.py#L710) - - [x] [KMeans](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/kmeans.py) - - [x] [PCA](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/pca.py) - -## Feedback - -If you have any feedback, please feel free to create an issue pointing out whatever you want or reach out to me at rgvieira97@gmail.com +## Roadmap -## Contributing +Check the `examples` folder to see the roadmap, some known issues, and ideas for enhancing the code. -Contributions are what makes the open-source community such an amazing place to learn, inspire, and create. Any contributions you make are greatly appreciated. +

(back to top)

-If you have a suggestion that would make this better, please read carefully the [Contributing Guide](https://github.com/rafaelgreca/scratchml/blob/main/docs/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rafaelgreca/scratchml/blob/main/docs/CODE_OF_CONDUCT.md) before contributing. +## Contributing -## Acknowledge +Contributions are what makes the open-source community such an amazing place to learn, inspire, and create. Any contributions you make are greatly appreciated. If you have a suggestion that would make this better, please read carefully the [Contributing Guide](https://github.com/rafaelgreca/scratchml/blob/main/docs/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rafaelgreca/scratchml/blob/main/docs/CODE_OF_CONDUCT.md) before contributing. -We would like to thank all these amazing materials and repositories for their amazing work, which indirectly contributed in some sort or that inspired us to create this project. +If you agree with the [Code of Conduct](https://github.com/rafaelgreca/scratchml/blob/main/docs/CODE_OF_CONDUCT.md), read carefully the [Contributing Guide](https://github.com/rafaelgreca/scratchml/blob/main/docs/CONTRIBUTING.md), and still want to contribute, here's a step-by-step of how you contribute (you can also simply open an issue if you don't want to code anything): -- [REPOSITORY] [SKADI by Douglas Oliveira](https://github.com/Dellonath/SKADI/) -- [REPOSITORY] [ML From Scratch by Erik Linder-Norén](https://github.com/eriklindernoren/ML-From-Scratch) -- [REPOSITORY] [Machine Learning from Scratch by AssemblyAI](https://github.com/AssemblyAI-Community/Machine-Learning-From-Scratch) -- [COURSE] [Machine Learning Specialization by Andrew Ng](https://www.coursera.org/specializations/machine-learning-introduction) -- [COURSE] [Machine Learning From Scratch by AssemblyAI](https://www.youtube.com/watch?v=p1hGz0w_OCo&list=PLcWfeUsAys2k_xub3mHks85sBHZvg24Jd) +1. Fork the project +2. Create a branch in your forked repository using a `feature` tag if you are implementing a new feature or a `bugfix` tag if you are fixing a issue/bug (examples: `git checkout -b feature/AmazingFeature` or `git checkout -b bugfix/FixingBugX`) +3. Commit your changes (`git commit -m `) +4. Push to the branch (`git push origin `) +5. Open a pull request +

(back to top)

## License Distributed under the [MIT](https://choosealicense.com/licenses/mit/) License. See LICENSE for more information. +

(back to top)

+ ## Authors A huge shoutout to everyone who contributed to the success of the project. [Check everyone here!](https://github.com/rafaelgreca/scratchml/blob/main/docs/AUTHORS.md). + +

(back to top)

+ +## Acknowledgments + +We would like to thank all these amazing materials and repositories for their amazing work, which indirectly contributed to or inspired us to create this project. + +- [REPOSITORY] [SKADI by Douglas Oliveira](https://github.com/Dellonath/SKADI/) +- [REPOSITORY] [ML From Scratch by Erik Linder-Norén](https://github.com/eriklindernoren/ML-From-Scratch) +- [REPOSITORY] [Machine Learning from Scratch by AssemblyAI](https://github.com/AssemblyAI-Community/Machine-Learning-From-Scratch) +- [COURSE] [Machine Learning Specialization by Andrew Ng](https://www.coursera.org/specializations/machine-learning-introduction) +- [COURSE] [Machine Learning From Scratch by AssemblyAI](https://www.youtube.com/watch?v=p1hGz0w_OCo&list=PLcWfeUsAys2k_xub3mHks85sBHZvg24Jd) + +

(back to top)

diff --git a/docs/AUTHORS.md b/docs/AUTHORS.md index 3e37dc2..47fddfb 100644 --- a/docs/AUTHORS.md +++ b/docs/AUTHORS.md @@ -3,26 +3,21 @@ Primary Authors * __[Rafael Greca](https://www.github.com/rafaelgreca)__ - @rafaelgreca is the current maintainer of the code and has written much of the - current code base. + @rafaelgreca is the current maintainer of the code and has written much of the current code base. Other Contributors ================== -The incomplete list of individuals below have provided patches or otherwise -contributed to the project prior to the project being hosted on GitHub. See the -GitHub commit log for a list of recent contributors. We would like to thank -everyone who has contributed to the project in any way. +The incomplete list of individuals below have provided patches or otherwise contributed to the project prior to the project being hosted on GitHub. See the GitHub commit log for a list of recent contributors. We would like to thank everyone who has contributed to the project in any way. * __[Rene Ivancak](https://github.com/Renkooo)__ - @Renkooo implemented a classification solution for the SVM model, - complete with testing and an example script. + @Renkooo implemented a classification solution for the Support Vector Classification (SVC) model, complete with testing and an example script. * __[Sebastian Ondrus](https://github.com/SebastianOndrus)__ - @SebastianOndrus has contributed to the project by implementing a Regressor for the Multilayer Perceptron model, including test and example script. + @SebastianOndrus has contributed to the project by implementing a Regressor for the Multilayer Perceptron model, including tests and example script. * __[Matus Pohorenec](https://github.com/MatusPohorenec)__ - @MatusPohorenec has contributed to the project by introducing support for SVR Classification in `svr.py`, including tests and a usage example. + @MatusPohorenec has contributed to the project by introducing support for Support Vector Regression (SVR), including tests and an usage example. diff --git a/docs/CODE_OF_CONDUCT.md b/docs/CODE_OF_CONDUCT.md index 1a22ad0..eef892a 100644 --- a/docs/CODE_OF_CONDUCT.md +++ b/docs/CODE_OF_CONDUCT.md @@ -1,112 +1,95 @@ + + # Code of Conduct - ScratchML ## Our Pledge -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to make participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, sex characteristics, gender identity and expression, -level of experience, education, socio-economic status, nationality, personal -appearance, race, religion, or sexual identity and orientation. +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. + +

(back to top)

## Our Standards -Examples of behaviour that contributes to a positive environment for our -community include: +Examples of behaviour that contributes to a positive environment for our community include: * Demonstrating empathy and kindness toward other people * Being respectful of differing opinions, viewpoints, and experiences * Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologising to those affected by our mistakes, - and learning from the experience -* Focusing on what is best not just for us as individuals, but for the - overall community +* Accepting responsibility and apologising to those affected by our mistakes, and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behaviour include: * The use of sexualised language or imagery, and sexual attention or advances * Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment -* Publishing others' private information, such as a physical or email - address, without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting +* Publishing others' private information, such as a physical or email address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +

(back to top)

## Our Responsibilities -Project maintainers are responsible for clarifying and enforcing our standards of -acceptable behaviour and will take appropriate and fair corrective action in -response to any behaviour that they deem inappropriate, -threatening, offensive, or harmful. +Project maintainers are responsible for clarifying and enforcing our standards of acceptable behaviour and will take appropriate and fair corrective action in response to any behaviour that they deem inappropriate, threatening, offensive, or harmful. -Project maintainers have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, and will -communicate reasons for moderation decisions when appropriate. +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. + +

(back to top)

## Scope -This Code of Conduct applies within all community spaces, and also applies when -an individual is officially representing the community in public spaces. -Examples of representing our community include using an official e-mail address, -posting via an official social media account, or acting as an appointed -representative at an online or offline event. +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + +

(back to top)

## Enforcement -Instances of abusive, harassing, or otherwise unacceptable behaviour may be -reported to the community leaders responsible for enforcement at . -All complaints will be reviewed and investigated promptly and fairly. +Instances of abusive, harassing, or otherwise unacceptable behaviour may be reported to the community leaders responsible for enforcement at . All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the reporter of any incident. -All community leaders are obligated to respect the privacy and security of the -reporter of any incident. +

(back to top)

## Enforcement Guidelines -Community leaders will follow these Community Impact Guidelines in determining -the consequences for any action they deem in violation of this Code of Conduct: +Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: + +

(back to top)

### 1. Correction -**Community Impact**: Use of inappropriate language or other behaviour deemed -unprofessional or unwelcome in the community. +**Community Impact**: Use of inappropriate language or other behaviour deemed unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behaviour was inappropriate. A public apology may be requested. -**Consequence**: A private, written warning from community leaders, providing -clarity around the nature of the violation and an explanation of why the -behaviour was inappropriate. A public apology may be requested. +

(back to top)

### 2. Warning -**Community Impact**: A violation through a single incident or series -of actions. +**Community Impact**: A violation through a single incident or series of actions. + +**Consequence**: A warning with consequences for continued behaviour. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. -**Consequence**: A warning with consequences for continued behaviour. No -interaction with the people involved, including unsolicited interaction with -those enforcing the Code of Conduct, for a specified period of time. This -includes avoiding interactions in community spaces as well as external channels -like social media. Violating these terms may lead to a temporary or -permanent ban. +

(back to top)

### 3. Temporary Ban -**Community Impact**: A serious violation of community standards, including -sustained inappropriate behaviour. +**Community Impact**: A serious violation of community standards, including sustained inappropriate behaviour. -**Consequence**: A temporary ban from any sort of interaction or public -communication with the community for a specified period of time. No public or -private interaction with the people involved, including unsolicited interaction -with those enforcing the Code of Conduct, is allowed during this period. -Violating these terms may lead to a permanent ban. +**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. + +

(back to top)

### 4. Permanent Ban -**Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behaviour, harassment of an -individual, or aggression toward or disparagement of classes of individuals. +**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behaviour, harassment of an individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the community. -**Consequence**: A permanent ban from any sort of public interaction within -the community. +

(back to top)

## Attribution This Code of Conduct is adapted from the [Contributor Covenant](https://contributor-covenant.org/), version [1.4](https://www.contributor-covenant.org/version/1/4/code-of-conduct/code_of_conduct.md) and [2.0](https://www.contributor-covenant.org/version/2/0/code_of_conduct/code_of_conduct.md), and was generated by [contributing-gen](https://github.com/bttger/contributing-gen). + +

(back to top)

diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 8959185..8d25179 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -1,3 +1,5 @@ + + # Contributing to ScratchML @@ -21,9 +23,6 @@ All types of contributions are encouraged and valued. See the [Table of Contents - [Suggesting Enhancements](#suggesting-enhancements) - [Your First Code Contribution](#your-first-code-contribution) - [Improving The Documentation](#improving-the-documentation) -- [Styleguides](#styleguides) - - [Commit Messages](#commit-messages) -- [Join The Project Team](#join-the-project-team) ## Code of Conduct @@ -33,6 +32,7 @@ This project and everyone participating in it is governed by the By participating, you are expected to uphold this code. Please report unacceptable behavior to . +

(back to top)

## I Have a Question @@ -46,11 +46,15 @@ If you then still feel the need to ask a question and need clarification, we rec We will then take care of the issue as soon as possible. +

(back to top)

+ ## I Want To Contribute > ### Legal Notice > When contributing to this project, you must agree that you have authored 100% of the content, that you have the necessary rights to the content and that the content you contribute may be provided under the project licence. +

(back to top)

+ ### Reporting Bugs @@ -64,6 +68,8 @@ A good bug report shouldn't leave others needing to chase you up for more inform - Also make sure to search the internet (including Stack Overflow) to see if users outside of the GitHub community have discussed the issue. - Collect information about the bug. +

(back to top)

+ #### How Do I Submit a Good Bug Report? @@ -82,13 +88,14 @@ Once it's filed: - A team member will try to reproduce the issue with your provided steps. If there are no reproduction steps or no obvious way to reproduce the issue, the team will ask you for those steps and mark the issue as `needs-repro`. Bugs with the `needs-repro` tag will not be addressed until they are reproduced. - If the team is able to reproduce the issue, it will be marked `needs-fix`, as well as possibly other tags (such as `critical`), and the issue will be left to be [implemented by someone](#your-first-code-contribution). - - +

(back to top)

### Suggesting Enhancements This section guides you through submitting an enhancement suggestion for ScratchML, **including completely new features and minor improvements to existing functionality**. Following these guidelines will help maintainers and the community to understand your suggestion and find related suggestions. +

(back to top)

+ #### Before Submitting an Enhancement @@ -97,6 +104,8 @@ This section guides you through submitting an enhancement suggestion for Scratch - Perform a [search](https://github.com/rafaelgreca/scratchml/issues) to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one. - Find out whether your idea fits with the scope and aims of the project. It's up to you to make a strong case to convince the project's developers of the merits of this feature. Keep in mind that we want features that will be useful to the majority of our users and not just a small subset. If you're just targeting a minority of users, consider writing an add-on/plugin library. +

(back to top)

+ #### How Do I Submit a Good Enhancement Suggestion? @@ -110,6 +119,9 @@ Enhancement suggestions are tracked as [GitHub issues](https://github.com/rafael - **Explain why this enhancement would be useful** to most ScratchML users. You may also want to point out the other projects that solved it better and which could serve as inspiration. - Use the [Bug Report](https://github.com/rafaelgreca/scratchml/tree/main/.github/ISSUE_TEMPLATE/BUG_REPORT.md) or the [Feature Request](https://github.com/rafaelgreca/scratchml/tree/main/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md) template. - +

(back to top)

+ ## Attribution This guide is based on the **contributing-gen**. [Make your own](https://github.com/bttger/contributing-gen)! + +

(back to top)

diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md new file mode 100644 index 0000000..da801e4 --- /dev/null +++ b/docs/ROADMAP.md @@ -0,0 +1,106 @@ + + +# Roadmap + +## Implementations + +- [x] Scalers + - [x] [StandardScaler](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/scalers.py#L155) + - [x] [MinMaxScaler](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/scalers.py#L37) +- [ ] Regularizations + - [x] [L1](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/regularizations.py#L4) + - [x] [L2](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/regularizations.py#L27) + - [ ] Batch Normalization +- [x] Activation functions + - [x] [Sigmoid](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L109) + - [x] [ReLU](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L23) + - [x] [Linear](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L4) + - [x] [Softmax](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L132) + - [x] [TanH](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L84) + - [x] [ELU](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L42) + - [x] [Leaky ReLU](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L65) + - [x] [SoftPlus](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L157) + - [x] [SELU](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/activations.py#L180) +- [x] Loss functions + - [x] [Binary Cross Entropy](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/losses.py#L4) + - [x] [Cross Entropy](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/losses.py#L33) +- [x] Metrics + - [x] Regression Metrics + - [x] [Mean Squared Error (MSE)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L7) + - [x] [Root Mean Squared Error (RMSE)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L29) + - [x] [Mean Absolute Error (MAE)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L51) + - [x] [Median Absolute Error (MedAE)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L73) + - [x] [Mean Absolute Percentage Error (MAPE)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L95) + - [x] [Mean Squared Logarithmic Error (MSLE)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L128) + - [x] [Max Error (ME)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L156) + - [x] [R Squared (R2)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L180) + - [x] Classification Metrics + - [x] [Accuracy](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L200) + - [x] [Precision](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L215) + - [x] [Recall](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L272) + - [x] [F1-Score](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L329) + - [x] [Confusion Matrix](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L373) + - [x] [ROC AUC Score](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L474) + - [x] [False Positive Rate (FPR)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L458) + - [x] [True Positive Rate (TPR)](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/metrics.py#L442) +- [x] Distances + - [x] [Euclidean](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/distances.py#L6) + - [x] [Manhattan](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/distances.py#L26) + - [x] [Chebyshev](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/distances.py#L46) + - [x] [Minkowski](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/distances.py#L66) +- [x] Encoders + - [x] [One-hot Encoding](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/encoders.py#L133) + - [x] [Label Encoding](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/encoders.py#L39) +- [x] Splitters + - [x] [KFold](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/utils.py#L42) + - [x] [Stratify KFold](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/utils.py#L42) + - [x] [Train Test Split](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/utils.py#L187) + - [x] [Split Into Batches](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/utils.py#L5) +- [ ] Models + - [ ] Isolation Forest + - [ ] AdaBoost + - [ ] AdaBoost Regressor + - [ ] AdaBoost Classifier + - [x] [Linear Regression](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/linear_regression.py) + - [x] [Logistic Regression](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/logistic_regression.py) + - [x] SVM + - [x] [SVC](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/svc.py) + - [x] [SVR](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/svr.py) + - [x] KNN + - [x] [KNN Classifier](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/knn.py#L236) + - [x] [KNN Regressor](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/knn.py#L375) + - [x] [Naive Bayes](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/naive_bayes.py) + - [x] Random Forest + - [x] [Random Forest Classifier](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/random_forest.py#L291) + - [x] [Random Forest Regressor](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/random_forest.py#L445) + - [x] Decision Tree + - [x] [Decision Tree Classifier](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/decision_tree.py#L525) + - [x] [Decision Tree Regressor](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/decision_tree.py#L640) + - [x] [Perceptron](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/perceptron.py) + - [x] MLP + - [x] [MLP Classifier](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/multilayer_perceptron.py#L569) + - [x] [MLP Regressor](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/multilayer_perceptron.py#L710) + - [x] [KMeans](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/kmeans.py) + - [x] [PCA](https://github.com/rafaelgreca/scratchml/blob/main/scratchml/models/pca.py) + +

(back to top)

+ +## Known Issues + +- [ ] Binary Labeling Issue in Perceptron Model +- [ ] CI Unit test workflow not working (Numpy version error) +- [ ] MLP not working properly when dealing with a multiclass classification problem (I think it's a vanishing gradient problem) +- [ ] Sometimes the Logistic Regression model gets stuck (I think it's a vanishing gradient problem) +- [ ] Recursion error when max depth is None in the Decision Tree model +- [ ] Zero Division Warning in the Decision Tree code + +

(back to top)

+ +## Enhancement + +- [ ] Create visualization plots for all models (including for the training step) +- [ ] Create a function to print the Decision Tree and Random Forest Tree +- [ ] Optimize KNN code (looping functions and distance metrics taking too long) +- [ ] Improve testings (exclude redudant tests and add tests for model parameters) + +

(back to top)

From 5ed1e8868e6080c96dd7e580a45c6a75951eeb97 Mon Sep 17 00:00:00 2001 From: Rafael Greca Date: Tue, 3 Dec 2024 19:24:28 -0300 Subject: [PATCH 2/4] gruping svm implementations into one file --- README.md | 10 +- examples/svc.py | 68 -- examples/svm.py | 93 +++ examples/svr.py | 33 - scratchml/metrics.py | 4 +- scratchml/models/multilayer_perceptron.py | 16 +- scratchml/models/svc.py | 344 ---------- scratchml/models/svm.py | 757 ++++++++++++++++++++++ scratchml/models/svr.py | 234 ------- tests/models/test_mlp.py | 4 +- tests/models/{test_svc.py => test_svm.py} | 139 +++- tests/models/test_svr.py | 135 ---- 12 files changed, 1005 insertions(+), 832 deletions(-) delete mode 100644 examples/svc.py create mode 100644 examples/svm.py delete mode 100644 examples/svr.py delete mode 100644 scratchml/models/svc.py create mode 100644 scratchml/models/svm.py delete mode 100644 scratchml/models/svr.py rename tests/models/{test_svc.py => test_svm.py} (56%) delete mode 100644 tests/models/test_svr.py diff --git a/README.md b/README.md index 5fec82a..019e7d9 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,9 @@ Algorithms: - Perceptron (added in version 4.0.0) - PCA (added in version 8.0.0) - Random Forest Classifier and Random Forest Regressor (added in version 7.0.0) -- Support Vector Classifier and Support Vector Regressor (added in version 10.0.0) +- Support Vector Classifier** and Support Vector Regressor (added in version 10.0.0) + +** only available for binary classification at the moment Data split functions: @@ -149,6 +151,12 @@ Encoders: - Label Encoding (added in version 1.0.0) - One-hot Encoding (added in version 1.0.0) +Kernels: + +- Linear (added in version 10.0.0) +- Polynomial (added in version 10.0.0) +- RBF (added in version 10.0.0) + Loss functions: - Binary Cross Entropy (added in version 9.0.0) diff --git a/examples/svc.py b/examples/svc.py deleted file mode 100644 index 2a53634..0000000 --- a/examples/svc.py +++ /dev/null @@ -1,68 +0,0 @@ -from scratchml.models.svc import SVC -from sklearn.datasets import make_classification -from scratchml.utils import train_test_split -from sklearn.metrics import ( - accuracy_score, - f1_score, - precision_score, - recall_score, - roc_auc_score, -) -import numpy as np - - -def example_svc(): - """ - Example of how to use the optimized SVC model with enhanced metrics, - polynomial kernel, and cross-validation for high precision. - """ - X, y = make_classification( - n_samples=2000, - n_features=20, - n_classes=2, - n_informative=15, - n_redundant=5, - class_sep=1.8, - random_state=42, - ) - y = np.where(y == 0, -1, 1) # Convert labels to -1 and 1 for SVM compatibility - - # Split the dataset into training and test sets - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) - - # Initialize the SVC model with optimal parameters - model = SVC( - C=0.4, - alpha=0.015, - kernel="linear", - degree=4, - max_iter=1000, - tol=1e-5, - learning_rate=5e-4, - decay=0.995, - batch_size=16, - adaptive_lr=True, - ) - - # Train the model - model.fit(X_train, y_train) - - # Evaluate the model on the test set - y_pred = model.predict(X_test) - accuracy = accuracy_score(y_test, y_pred) - f1 = f1_score(y_test, y_pred) - precision = precision_score(y_test, y_pred) - recall = recall_score(y_test, y_pred) - roc_auc = roc_auc_score(y_test, y_pred) - - # Print evaluation metrics - print("Test Set Results:") - print(f"Accuracy: {accuracy * 100:.2f}%") - print(f"Precision: {precision * 100:.2f}%") - print(f"Recall: {recall * 100:.2f}%") - print(f"F1 Score: {f1:.2f}") - print(f"ROC-AUC Score: {roc_auc:.2f}") - - -if __name__ == "__main__": - example_svc() diff --git a/examples/svm.py b/examples/svm.py new file mode 100644 index 0000000..8e5be73 --- /dev/null +++ b/examples/svm.py @@ -0,0 +1,93 @@ +from sklearn.datasets import make_classification, make_regression +from scratchml.models.svm import SVC, SVR +from scratchml.utils import train_test_split +from scratchml.metrics import ( + accuracy, + f1_score, + precision, + recall, +) +import numpy as np + + +def example_svc(): + """ + Example of how to use the optimized SVC model with enhanced metrics, + polynomial kernel, and cross-validation for high precision. + """ + X, y = make_classification( + n_samples=2000, + n_features=20, + n_classes=2, + n_informative=15, + n_redundant=5, + class_sep=1.8, + random_state=42, + ) + y = np.where(y == 0, -1, 1) # Convert labels to -1 and 1 for SVM compatibility + + # Split the dataset into training and test sets + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) + + # Initialize the SVC model with optimal parameters + model = SVC( + C=0.4, + alpha=0.015, + kernel="linear", + degree=4, + max_iter=1000, + tol=1e-5, + learning_rate=5e-4, + decay=0.995, + batch_size=16, + adaptive_lr=True, + gamma="scale", + ) + + # Train the model + model.fit(X_train, y_train) + + # Evaluate the model on the test set + y_pred = model.predict(X_test) + accuracy_score = accuracy(y_test, y_pred) + f1 = f1_score(y_test, y_pred) + precision_score = precision(y_test, y_pred) + recall_score = recall(y_test, y_pred) + + # Print evaluation metrics + print("Test Set Results:") + print(f"Accuracy: {accuracy_score * 100:.2f}%") + print(f"Precision: {precision_score * 100:.2f}%") + print(f"Recall: {recall_score * 100:.2f}%") + print(f"F1 Score: {f1:.2f}\n") + + +def example_svr() -> None: + """ + Practical example of how to use the Support Vector Regression (SVR) model. + """ + # generating a dataset for the regression task + X, y = make_regression( + n_samples=2000, n_features=5, n_targets=1, shuffle=True, noise=30 + ) + + # splitting the data into training and test + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.15, shuffle=True, stratify=False + ) + + # creating a SVR model + svr = SVR(kernel="linear", C=1.0, epsilon=0.1) + + # fitting the model + svr.fit(X=X_train, y=y_train) + + # assessing the model's performance + score = svr.score(X=X_test, y=y_test, metric="r_squared") + + print(f"The model achieved a R² score of {score}.\n") + + +if __name__ == "__main__": + example_svc() + example_svr() diff --git a/examples/svr.py b/examples/svr.py deleted file mode 100644 index 1649fab..0000000 --- a/examples/svr.py +++ /dev/null @@ -1,33 +0,0 @@ -from scratchml.models.svr import BaseSVR -from scratchml.utils import train_test_split -from sklearn.datasets import make_regression - - -def example_svr() -> None: - """ - Practical example of how to use the Support Vector Regression (SVR) model. - """ - # generating a dataset for the regression task - X, y = make_regression( - n_samples=2000, n_features=5, n_targets=1, shuffle=True, noise=30 - ) - - # splitting the data into training and test - X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=0.15, shuffle=True, stratify=False - ) - - # creating a linear regression model - svr = BaseSVR(kernel="linear", C=1.0, epsilon=0.1) - - # fitting the model - svr.fit(X=X_train, y=y_train) - - # assessing the model's performance - score = svr.score(X=X_test, y=y_test, metric="r_squared") - - print(f"The model achieved a R² score of {score}.\n") - - -if __name__ == "__main__": - example_svr() diff --git a/scratchml/metrics.py b/scratchml/metrics.py index f53bf96..2cbe3aa 100644 --- a/scratchml/metrics.py +++ b/scratchml/metrics.py @@ -472,7 +472,7 @@ def false_positive_rate(y: np.ndarray, y_hat: np.ndarray) -> np.float32: def roc_auc_score( - y: np.ndarray, y_hat: np.ndarray, average: str = "micro" + y: np.ndarray, y_hat: np.ndarray, average: str = "binary" ) -> np.float32: """ Calculates the False Positive Rate (FPR). @@ -486,7 +486,7 @@ def roc_auc_score( Returns: np.float32: the value of the FPR. """ - _valid_averages = ["micro", "macro", "weighted"] + _valid_averages = ["micro", "macro", "weighted", "binary"] # validating the average value try: diff --git a/scratchml/models/multilayer_perceptron.py b/scratchml/models/multilayer_perceptron.py index d6438c9..87d48a5 100644 --- a/scratchml/models/multilayer_perceptron.py +++ b/scratchml/models/multilayer_perceptron.py @@ -731,13 +731,17 @@ def __init__( Args: loss_function (str, optional): The loss function that will be used. Defaults to "mse". - hidden_layer_sizes (Tuple[int, ...], optional): The sizes of the hidden layers. Defaults to (100,). - activation (str, optional): The activation function that will be used. Defaults to "relu". + hidden_layer_sizes (Tuple[int, ...], optional): The sizes of the hidden layers. + Defaults to (100,). + activation (str, optional): The activation function that will be used. + Defaults to "relu". alpha (float, optional): The strength of the L2 regularization term. Defaults to 0.0001. momentum (float, optional): The momentum for gradient descent update. Defaults to 0.9. batch_size (Union[int, str], optional): The batch size. Defaults to "auto". - learning_rate (str, optional): Learning rate schedule for weight updates. Defaults to "constant". - learning_rate_init (float, optional): The initial value for the learning rate. Defaults to 0.001. + learning_rate (str, optional): Learning rate schedule for weight updates. + Defaults to "constant". + learning_rate_init (float, optional): The initial value for the learning rate. + Defaults to 0.001. max_iter (int, optional): The number of max iterations. Defaults to 200. tol (float, optional): The tolerance for optimization. Defaults to 1e-4. verbose (int, optional): Whether to print progress messages. Defaults to 0. @@ -844,7 +848,3 @@ def score( if metric == "max_error": return max_error(y, y_hat) - - raise ValueError( - f"Invalid value for 'metric'. Must be one of {self._valid_score_metrics}." - ) diff --git a/scratchml/models/svc.py b/scratchml/models/svc.py deleted file mode 100644 index 5de2054..0000000 --- a/scratchml/models/svc.py +++ /dev/null @@ -1,344 +0,0 @@ -from scratchml.utils import split_data_into_batches -from ..metrics import accuracy, recall, precision, f1_score, confusion_matrix -from scratchml.regularizations import l2 -from scratchml.scalers import StandardScaler -import numpy as np -from typing import List, Union - - -class SVC: - """ - Support Vector Classifier (SVC) with options for linear, polynomial, and RBF kernels. - """ - - def __init__( - self, - C: float = 0.35, - alpha: float = 0.01, - kernel: str = "linear", - degree: int = 3, - max_iter: int = 1500, - tol: float = 1e-4, - learning_rate: float = 1e-5, - decay: float = 0.999, - batch_size: int = 64, - early_stopping: bool = True, - adaptive_lr: bool = False, - verbose: int = 0, - ) -> None: - """ - Creates a Support Vector Classifier (SVC) instance. - - Args: - C (float): Regularization parameter. Defaults to 0.35. - alpha (float): Learning rate for gradient descent. Defaults to 0.01. - kernel (str): Kernel type to be used in the algorithm. Valid values are "linear", "polynomial", and "rbf". Defaults to "linear". - degree (int): Degree of the polynomial kernel function ('poly'). Ignored by other kernels. Defaults to 3. - max_iter (int): Maximum number of iterations for training. Defaults to 1500. - tol (float): Tolerance for stopping criteria. Defaults to 1e-4. - learning_rate (float): Initial learning rate for gradient descent. Defaults to 1e-5. - decay (float): Learning rate decay factor. Defaults to 0.999. - batch_size (int): Size of mini-batches for stochastic gradient descent. Defaults to 64. - early_stopping (bool): Whether to use early stopping to terminate training when validation score is not improving. Defaults to True. - adaptive_lr (bool): Whether to use adaptive learning rate. Defaults to False. - verbose (int): Level of verbosity. 0 means no information, 1 means convergence information, 2 means detailed information. Defaults to 0. - """ - self.C = C - self.alpha = alpha - self.kernel = kernel - self.degree = degree - self.max_iter = max_iter - self.tol = tol - self.learning_rate = learning_rate - self.decay = decay - self.batch_size = batch_size - self.early_stopping = early_stopping - self.adaptive_lr = adaptive_lr - self.verbose = verbose - self.weights = None - self.classes_ = None - self.bias = 0 - self.scaler = StandardScaler() - self._valid_metrics = [ - "accuracy", - "precision", - "recall", - "f1_score", - "confusion_matrix", - ] - self._validate_parameters() - - def _apply_kernel(self, X1: np.ndarray, X2: np.ndarray) -> np.ndarray: - """ - Applies the selected kernel function. - - Args: - X1 (np.ndarray): First input array. - X2 (np.ndarray): Second input array. - - Returns: - np.ndarray: Result of applying the kernel function. - """ - if self.kernel == "linear": - return np.dot(X1, X2.T) - elif self.kernel == "polynomial": - return (1 + np.dot(X1, X2.T)) ** self.degree - elif self.kernel == "rbf": - gamma = 1 / X1.shape[1] - return np.exp( - -gamma * np.square(np.linalg.norm(X1[:, np.newaxis] - X2, axis=2)) - ) - - def _validate_parameters(self) -> None: - """ - Validates the parameters of the SVC instance. - - Raises: - ValueError: If any parameter is invalid. - """ - try: - assert isinstance(self.C, float) and self.C > 0 - except AssertionError as error: - raise ValueError( - "Regularization parameter C must be a positive float." - ) from error - - try: - assert isinstance(self.alpha, float) and self.alpha > 0 - except AssertionError as error: - raise ValueError("Learning rate alpha must be a positive float.") from error - - try: - assert self.kernel in ["linear", "polynomial", "rbf"] - except AssertionError as error: - raise ValueError( - "Kernel must be one of 'linear', 'polynomial', or 'rbf'." - ) from error - - try: - assert isinstance(self.degree, int) and self.degree > 0 - except AssertionError as error: - raise ValueError("Degree must be a positive integer.") from error - - try: - assert isinstance(self.max_iter, int) and self.max_iter > 0 - except AssertionError as error: - raise ValueError( - "Maximum number of iterations must be a positive integer." - ) from error - - try: - assert isinstance(self.tol, float) and self.tol > 0 - except AssertionError as error: - raise ValueError("Tolerance must be a positive float.") from error - - try: - assert isinstance(self.learning_rate, float) and self.learning_rate > 0 - except AssertionError as error: - raise ValueError("Learning rate must be a positive float.") from error - - try: - assert isinstance(self.decay, float) and 0 < self.decay <= 1 - except AssertionError as error: - raise ValueError("Decay must be a float between 0 and 1.") from error - - try: - assert isinstance(self.batch_size, int) and self.batch_size > 0 - except AssertionError as error: - raise ValueError("Batch size must be a positive integer.") from error - - try: - assert isinstance(self.early_stopping, bool) - except AssertionError as error: - raise ValueError("Early stopping must be a boolean.") from error - - try: - assert isinstance(self.adaptive_lr, bool) - except AssertionError as error: - raise ValueError("Adaptive learning rate must be a boolean.") from error - - try: - assert isinstance(self.weights, (type(None), np.ndarray)) - except AssertionError as error: - raise ValueError("Weights must be None or a numpy array.") from error - - try: - assert isinstance(self.classes_, (type(None), np.ndarray)) - except AssertionError as error: - raise ValueError("Classes must be None or a numpy array.") from error - - try: - assert isinstance(self.bias, (int, float)) - except AssertionError as error: - raise ValueError("Bias must be an integer or float.") from error - - try: - assert isinstance(self.scaler, StandardScaler) - except AssertionError as error: - raise ValueError("Scaler must be an instance of StandardScaler.") from error - - try: - assert isinstance(self._valid_metrics, list) and all( - isinstance(metric, str) for metric in self._valid_metrics - ) - except AssertionError as error: - raise ValueError("Valid metrics must be a list of strings.") from error - - def _check_is_fitted(self) -> None: - """ - Checks if the model is fitted. - - Raises: - ValueError: If the model is not trained. - """ - if self.weights is None: - raise ValueError("Model must be trained before prediction.") - - def fit(self, X: np.ndarray, y: np.ndarray) -> None: - """ - Trains the SVC model using mini-batch gradient descent. - - Args: - X (np.ndarray): Training data of shape (n_samples, n_features). - y (np.ndarray): Target labels of shape (n_samples,). - """ - # Identify unique classes in the target labels - self.classes_ = np.unique(y) - if len(self.classes_) == 2: - # Convert binary class labels to -1 and 1 - y_ = np.where(y == self.classes_[0], -1, 1) - else: - y_ = y.copy() - - # Standardize the features - X = self.scaler.fit_transform(X) - _, n_features = X.shape - # Initialize weights and bias - self.weights = np.zeros(n_features) - - no_improvement_count = 0 - for iteration in range(self.max_iter): - # Adjust learning rate based on decay - lr = self.learning_rate * (self.decay**iteration) - avg_update_norm = 0 - # Split data into mini-batches and shuffle - for X_batch, y_batch in split_data_into_batches( - X, y_, self.batch_size, shuffle=True - ): - weight_update = np.zeros_like(self.weights) - bias_update = 0 - # Iterate over each instance in the mini-batch - for idx in range(X_batch.shape[0]): - instance = X_batch[idx : idx + 1] - margin = y_batch[idx] * ( - np.dot(instance.flatten(), self.weights) - self.bias - ) - - # Update weights and bias if the margin condition is not satisfied - if margin < 1: - weight_update += -self.C * y_batch[idx] * instance.flatten() - bias_update += -self.C * y_batch[idx] - - # Apply L2 regularization to the weight update - weight_update += self.alpha * l2(self.weights) - - # Adjust learning rate if adaptive learning rate is enabled - lr_adjusted = ( - lr / (1 + np.linalg.norm(weight_update)) if self.adaptive_lr else lr - ) - # Update weights and bias - self.weights -= lr_adjusted * weight_update - self.bias -= lr_adjusted * bias_update - avg_update_norm += np.linalg.norm(weight_update) - - # Calculate average norm of weight updates - avg_update_norm /= X.shape[0] / self.batch_size - # Check for early stopping based on tolerance - if self.early_stopping and avg_update_norm < self.tol: - no_improvement_count += 1 - if ( - no_improvement_count > 10 - ): # Terminate if no improvement for 10 iterations - if self.verbose > 0: - print(f"Converged after {iteration} iterations.") - break - else: - no_improvement_count = 0 - - # Print detailed information every 100 iterations if verbose level is 2 - if self.verbose == 2 and iteration % 100 == 0: - print( - f"Iteration {iteration}: Avg norm of batch updates = {avg_update_norm}" - ) - - # Print final weight norm and bias if verbose level is greater than 0 - if self.verbose > 0: - print(f"Final weight norm: {np.linalg.norm(self.weights)}") - print(f"Final bias: {self.bias}") - - def predict(self, X: np.ndarray) -> np.ndarray: - """ - Predicts class labels for input data. - - Args: - X (np.ndarray): Input data of shape (n_samples, n_features). - - Returns: - np.ndarray: Predicted class labels. - - Raises: - ValueError: If the model is not trained. - """ - if self.weights is None: - raise ValueError("Model must be trained before prediction.") - X = self.scaler.transform(X) - linear_output = np.dot(X, self.weights) - self.bias - predictions = np.sign(linear_output) - - if len(self.classes_) == 2: - return np.where(predictions == -1, self.classes_[0], self.classes_[1]) - else: - return predictions - - def score( - self, - X: np.ndarray, - y: np.ndarray, - metric: str = "accuracy", - labels_cm: List[int] = None, - normalize_cm: bool = False, - ) -> Union[np.float64, np.ndarray]: - """ - Calculates the score of the model on a given dataset using the specified metric. - - Args: - X (np.ndarray): Test data of shape (n_samples, n_features). - y (np.ndarray): True labels of shape (n_samples,). - metric (str): Metric to use for evaluation ("accuracy", "precision", "recall", "f1_score", "confusion_matrix"). Defaults to "accuracy". - labels_cm (List[int], optional): Labels for confusion matrix computation, ignored for other metrics. Defaults to None. - normalize_cm (bool, optional): Whether to normalize the confusion matrix. Defaults to False. - - Returns: - Union[float, np.ndarray]: Computed score based on the specified metric. - - Raises: - ValueError: If the specified metric is not valid. - """ - self._check_is_fitted() - if metric not in self._valid_metrics: - raise ValueError(f"Invalid metric. Must be one of {self._valid_metrics}.") - - predictions = self.predict(X) - - if metric == "accuracy": - return accuracy(y, predictions) - elif metric == "precision": - return precision(y, predictions) - elif metric == "recall": - return recall(y, predictions) - elif metric == "f1_score": - return f1_score(y, predictions) - elif metric == "confusion_matrix": - return confusion_matrix( - y, predictions, labels=labels_cm, normalize=normalize_cm - ) diff --git a/scratchml/models/svm.py b/scratchml/models/svm.py new file mode 100644 index 0000000..3ae1b53 --- /dev/null +++ b/scratchml/models/svm.py @@ -0,0 +1,757 @@ +from abc import ABC +from cvxopt import matrix, solvers +from scratchml.utils import split_data_into_batches +from scratchml.regularizations import l2 +from scratchml.scalers import StandardScaler +from typing import List, Union +from ..metrics import ( + accuracy, + recall, + precision, + f1_score, + confusion_matrix, + mean_squared_error, + root_mean_squared_error, + r_squared, + mean_absolute_error, + median_absolute_error, + mean_absolute_percentage_error, + mean_squared_logarithmic_error, + max_error, +) +from ..kernels import rbf_kernel, linear_kernel, polynomial_kernel +from ..utils import convert_array_numpy +import numpy as np + + +class SVMBase(ABC): + """ + Creates a base class for the Support Vector Machine (SVM) model. + """ + + def __init__( + self, + C: float = 0.35, + alpha: float = 0.01, + kernel: str = "linear", + degree: int = 3, + max_iter: int = 1500, + tol: float = 1e-4, + learning_rate: float = 1e-5, + decay: float = 0.999, + batch_size: int = 64, + early_stopping: bool = True, + adaptive_lr: bool = False, + verbose: int = 0, + gamma: str = "scale", + ) -> None: + """ + Creates a Support Vector Classifier (SVC) instance. + + Args: + C (float): Regularization parameter. Defaults to 0.35. + alpha (float): Learning rate for gradient descent. Defaults to 0.01. + kernel (str): Kernel type to be used in the algorithm. Valid values are "linear", + "polynomial", and "rbf". Defaults to "linear". + degree (int): Degree of the polynomial kernel function ('poly'). + Ignored by other kernels. Defaults to 3. + max_iter (int): Maximum number of iterations for training. Defaults to 1500. + tol (float): Tolerance for stopping criteria. Defaults to 1e-4. + learning_rate (float): Initial learning rate for gradient descent. Defaults to 1e-5. + decay (float): Learning rate decay factor. Defaults to 0.999. + batch_size (int): Size of mini-batches for stochastic gradient descent. Defaults to 64. + early_stopping (bool): Whether to use early stopping to terminate training when + validation score is not improving. Defaults to True. + adaptive_lr (bool): Whether to use adaptive learning rate. Defaults to False. + verbose (int): Level of verbosity. 0 means no information, 1 means convergence + information, 2 means detailed information. Defaults to 0. + gamma (str, optional): Kernel coefficient for ‘rbf’, ‘poly’, + and ‘sigmoid’. Defaults to "scale". + """ + self.C = C + self.alpha = alpha + self.kernel = kernel + self.degree = degree + self.max_iter = max_iter + self.tol = tol + self.learning_rate = learning_rate + self.decay = decay + self.batch_size = batch_size + self.early_stopping = early_stopping + self.adaptive_lr = adaptive_lr + self.verbose = verbose + self.gamma = gamma + self._valid_kernels = ["linear", "polynomial", "rbf"] + self._valid_gammas = ["scale", "auto"] + + # setting the valid criterions for svc and svr + if isinstance(self, SVC): + self._valid_score_metrics = [ + "accuracy", + "precision", + "recall", + "f1_score", + "confusion_matrix", + ] + elif isinstance(self, SVR): + self._valid_score_metrics = [ + "r_squared", + "mse", + "mae", + "rmse", + "medae", + "mape", + "msle", + "max_error", + ] + + def fit(self, X: np.ndarray, y: np.ndarray) -> None: + """ + Function responsible for fitting the Decision Tree model. + + Args: + X (np.ndarray): the features array. + y (np.ndarray): the classes array. + """ + + def score( + self, X: np.ndarray, y: np.ndarray, metric: str = "accuracy", **kwargs + ) -> Union[np.float64, np.ndarray]: + """ + Uses the trained model to predict the classes of a given + data points (also called features). + + Args: + X (np.ndarray): the features array. + y (np.ndarray): the labels array. + metric (string): which metric should be used to assess + the model's performance. Defaults to Accuracy. + + Returns: + (np.float32, np.ndarray): the score achieved by the model + or its confusion matrix. + """ + + def predict(self, X: np.ndarray) -> np.ndarray: + """ + Uses the trained model to predict the classes of a given + data points (also called features). + + Args: + X (np.ndarray): the features. + + Returns: + np.ndarray: the predicted classes. + """ + + def _validate_parameters(self) -> None: + """ + Validates the parameters of the SVC instance. + + Raises: + ValueError: If any parameter is invalid. + """ + + +class SVC(SVMBase): + """ + Support Vector Classifier (SVC) with options for linear, polynomial, and RBF kernels. + """ + + def __init__( + self, + C: float = 0.35, + alpha: float = 0.01, + kernel: str = "linear", + degree: int = 3, + max_iter: int = 1500, + tol: float = 1e-4, + learning_rate: float = 1e-5, + decay: float = 0.999, + batch_size: int = 64, + early_stopping: bool = True, + adaptive_lr: bool = False, + verbose: int = 0, + gamma: str = "scale", + ) -> None: + """ + Creates a Support Vector Classifier (SVC) instance. + + Args: + C (float): Regularization parameter. Defaults to 0.35. + alpha (float): Learning rate for gradient descent. Defaults to 0.01. + kernel (str): Kernel type to be used in the algorithm. Valid values are "linear", + "polynomial", and "rbf". Defaults to "linear". + degree (int): Degree of the polynomial kernel function ('poly'). + Ignored by other kernels. Defaults to 3. + max_iter (int): Maximum number of iterations for training. Defaults to 1500. + tol (float): Tolerance for stopping criteria. Defaults to 1e-4. + learning_rate (float): Initial learning rate for gradient descent. Defaults to 1e-5. + decay (float): Learning rate decay factor. Defaults to 0.999. + batch_size (int): Size of mini-batches for stochastic gradient descent. Defaults to 64. + early_stopping (bool): Whether to use early stopping to terminate training when + validation score is not improving. Defaults to True. + adaptive_lr (bool): Whether to use adaptive learning rate. Defaults to False. + verbose (int): Level of verbosity. 0 means no information, 1 means convergence + information, 2 means detailed information. Defaults to 0. + gamma (str, optional): Kernel coefficient for ‘rbf’, ‘poly’, + and ‘sigmoid’. Defaults to "scale". + """ + super().__init__( + C, + alpha, + kernel, + degree, + max_iter, + tol, + learning_rate, + decay, + batch_size, + early_stopping, + adaptive_lr, + verbose, + gamma, + ) + self.scaler = StandardScaler() + self.weights = None + self.classes_ = None + self.bias = 0 + + def _apply_kernel(self, X1: np.ndarray, X2: np.ndarray) -> np.ndarray: + """ + Applies the selected kernel function. + + Args: + X1 (np.ndarray): First input array. + X2 (np.ndarray): Second input array. + + Returns: + np.ndarray: Result of applying the kernel function. + """ + if self.kernel == "linear": + return linear_kernel(X1, X2) + + if self.kernel == "polynomial": + return polynomial_kernel(X1, X2, self.degree) + + if self.kernel == "rbf": + return rbf_kernel(X1, X2, gamma=self.gamma) + + def _check_is_fitted(self) -> None: + """ + Checks if the model is fitted. + + Raises: + ValueError: If the model is not trained. + """ + if self.weights is None: + raise ValueError("Model must be trained before prediction.") + + def _validate_parameters(self) -> None: + """ + Validates the parameters of the SVC instance. + + Raises: + ValueError: If any parameter is invalid. + """ + try: + assert isinstance(self.C, float) and self.C > 0 + except AssertionError as error: + raise ValueError( + "Regularization parameter C must be a positive float.\n" + ) from error + + try: + assert isinstance(self.alpha, float) and self.alpha > 0 + except AssertionError as error: + raise ValueError( + "Learning rate alpha must be a positive float.\n" + ) from error + + try: + assert self.kernel in self._valid_kernels + except AssertionError as error: + raise ValueError( + "Kernel must be one of 'linear', 'polynomial', or 'rbf'.\n" + ) from error + + try: + assert self.gamma in self._valid_gammas + except AssertionError as error: + raise ValueError("Gamma must be 'auto' or 'scale'.\n") from error + + try: + assert isinstance(self.degree, int) and self.degree > 0 + except AssertionError as error: + raise ValueError("Degree must be a positive integer.\n") from error + + try: + assert isinstance(self.max_iter, int) and self.max_iter > 0 + except AssertionError as error: + raise ValueError( + "Maximum number of iterations must be a positive integer.\n" + ) from error + + try: + assert isinstance(self.tol, float) and self.tol > 0 + except AssertionError as error: + raise ValueError("Tolerance must be a positive float.\n") from error + + try: + assert isinstance(self.learning_rate, float) and self.learning_rate > 0 + except AssertionError as error: + raise ValueError("Learning rate must be a positive float.\n") from error + + try: + assert isinstance(self.decay, float) and 0 < self.decay <= 1 + except AssertionError as error: + raise ValueError("Decay must be a float between 0 and 1.\n") from error + + try: + assert isinstance(self.batch_size, int) and self.batch_size > 0 + except AssertionError as error: + raise ValueError("Batch size must be a positive integer.\n") from error + + try: + assert isinstance(self.early_stopping, bool) + except AssertionError as error: + raise ValueError("Early stopping must be a boolean.\n") from error + + try: + assert isinstance(self.adaptive_lr, bool) + except AssertionError as error: + raise ValueError("Adaptive learning rate must be a boolean.\n") from error + + try: + assert isinstance(self.weights, (type(None), np.ndarray)) + except AssertionError as error: + raise ValueError("Weights must be None or a numpy array.\n") from error + + try: + assert isinstance(self.classes_, (type(None), np.ndarray)) + except AssertionError as error: + raise ValueError("Classes must be None or a numpy array.\n") from error + + try: + assert isinstance(self.bias, (int, float)) + except AssertionError as error: + raise ValueError("Bias must be an integer or float.\n") from error + + try: + assert isinstance(self.scaler, StandardScaler) + except AssertionError as error: + raise ValueError( + "Scaler must be an instance of StandardScaler.\n" + ) from error + + try: + assert isinstance(self._valid_score_metrics, list) and all( + isinstance(metric, str) for metric in self._valid_score_metrics + ) + except AssertionError as error: + raise ValueError("Valid metrics must be a list of strings.\n") from error + + def fit(self, X: np.ndarray, y: np.ndarray) -> None: + """ + Trains the SVC model using mini-batch gradient descent. + + Args: + X (np.ndarray): Training data of shape (n_samples, n_features). + y (np.ndarray): Target labels of shape (n_samples,). + """ + self._validate_parameters() + + X = convert_array_numpy(X) + y = convert_array_numpy(y) + + # Identify unique classes in the target labels + self.classes_ = np.unique(y) + + if len(self.classes_) == 2: + # Convert binary class labels to -1 and 1 + y_ = np.where(y == self.classes_[0], -1, 1) + else: + y_ = y.copy() + + # Standardize the features + X = self.scaler.fit_transform(X) + _, n_features = X.shape + + # Initialize weights and bias + self.weights = np.zeros(n_features) + + no_improvement_count = 0 + + for iteration in range(self.max_iter): + # Adjust learning rate based on decay + lr = self.learning_rate * (self.decay**iteration) + avg_update_norm = 0 + + # Split data into mini-batches and shuffle + for X_batch, y_batch in split_data_into_batches( + X, y_, self.batch_size, shuffle=True + ): + weight_update = np.zeros_like(self.weights) + bias_update = 0 + + # Iterate over each instance in the mini-batch + for idx in range(X_batch.shape[0]): + instance = X_batch[idx : idx + 1] + margin = y_batch[idx] * ( + np.dot(instance.flatten(), self.weights) - self.bias + ) + + # Update weights and bias if the margin condition is not satisfied + if margin < 1: + weight_update += -self.C * y_batch[idx] * instance.flatten() + bias_update += -self.C * y_batch[idx] + + # Apply L2 regularization to the weight update + weight_update += self.alpha * l2(self.weights) + + # Adjust learning rate if adaptive learning rate is enabled + lr_adjusted = ( + lr / (1 + np.linalg.norm(weight_update)) if self.adaptive_lr else lr + ) + + # Update weights and bias + self.weights -= lr_adjusted * weight_update + self.bias -= lr_adjusted * bias_update + avg_update_norm += np.linalg.norm(weight_update) + + # Calculate average norm of weight updates + avg_update_norm /= X.shape[0] / self.batch_size + + # Check for early stopping based on tolerance + if self.early_stopping and avg_update_norm < self.tol: + no_improvement_count += 1 + if ( + no_improvement_count > 10 + ): # Terminate if no improvement for 10 iterations + if self.verbose > 0: + print(f"Converged after {iteration} iterations.") + break + else: + no_improvement_count = 0 + + # Print detailed information every 100 iterations if verbose level is 2 + if self.verbose == 2 and iteration % 100 == 0: + print( + f"Iteration {iteration}: Avg norm of batch updates = {avg_update_norm}" + ) + + # Print final weight norm and bias if verbose level is greater than 0 + if self.verbose > 0: + print(f"Final weight norm: {np.linalg.norm(self.weights)}") + print(f"Final bias: {self.bias}") + + def predict(self, X: np.ndarray) -> np.ndarray: + """ + Predicts class labels for input data. + + Args: + X (np.ndarray): Input data of shape (n_samples, n_features). + + Returns: + np.ndarray: Predicted class labels. + + Raises: + ValueError: If the model is not trained. + """ + self._check_is_fitted() + + X = convert_array_numpy(X) + + X = self.scaler.transform(X) + linear_output = np.dot(X, self.weights) - self.bias + predictions = np.sign(linear_output) + + if len(self.classes_) == 2: + return np.where(predictions == -1, self.classes_[0], self.classes_[1]) + + return predictions + + def score( + self, + X: np.ndarray, + y: np.ndarray, + metric: str = "accuracy", + labels_cm: List = None, + normalize_cm: bool = False, + ) -> Union[np.float64, np.ndarray]: + """ + Uses the trained model to predict the classes of a given + data points (also called features). + + Args: + X (np.ndarray): the features array. + y (np.ndarray): the labels array. + metric (string): which metric should be used to assess + the model's performance. Defaults to Accuracy. + labels_cm (str, optional): which labels should be used to calculate + the confusion matrix. If other metric is selected, then this + parameter will be ignore. Defaults to None. + normalize_cm (bool, optional): whether the confusion matrix should be + normalized ('all', 'pred', 'true') or not. If other metric is selected, + then this parameter will be ignore. Defaults to False. + + Returns: + (np.float32, np.ndarray): the score achieved by the model + or its confusion matrix. + """ + self._check_is_fitted() + + try: + assert metric in self._valid_score_metrics + except AssertionError as error: + raise ValueError( + f"Invalid value for 'metric'. Must be {self._valid_score_metrics}.\n" + ) from error + + y_hat = self.predict(X) + + if metric == "accuracy": + return accuracy(y, y_hat) + + if metric == "precision": + return precision(y, y_hat) + + if metric == "recall": + return recall(y, y_hat) + + if metric == "f1_score": + return f1_score(y, y_hat) + + if metric == "confusion_matrix": + return confusion_matrix(y, y_hat, labels_cm, normalize_cm) + + +class SVR(SVMBase): + """ + Base class for Support Vector Regression (SVR). + """ + + def __init__( + self, + kernel: str = "rbf", + C: float = 1.0, + epsilon: float = 0.1, + degree: int = 3, + gamma: str = "scale", + ) -> None: + """ + Initializes SVR with default parameters. + + Args: + kernel (str, optional): Kernel type to be used in the algorithm. Defaults to "rbf". + C (float, optional): Regularization parameter. Defaults to 1.0. + epsilon (float, optional): Epsilon parameter in the epsilon-SVR model. Defaults to 0.1. + degree (int, optional): Degree of the polynomial kernel. + Ignored by other kernels. Defaults to 3. + gamma (str, optional): Kernel coefficient for ‘rbf’, ‘poly’, + and ‘sigmoid’. Defaults to "scale". + """ + super().__init__( + C=C, + kernel=kernel, + degree=degree, + gamma=gamma, + ) + self.C = C + self.epsilon = epsilon + self.X_ = None + self.y_ = None + self.alphas_ = None + self.b_ = None + self.K_ = None + + def _validate_parameters(self) -> None: + """ + Validates the parameters passed during initialization. + """ + if not isinstance(self.C, (int, float)) or self.C <= 0: + raise ValueError("C must be a positive number.\n") + + if self.kernel not in self._valid_kernels: + raise ValueError("Kernel must be one of 'linear', 'poly', or 'rbf'.\n") + + if not isinstance(self.epsilon, (int, float)) or self.epsilon < 0: + raise ValueError("Epsilon must be a non-negative number.\n") + + if not isinstance(self.degree, int) or self.degree <= 0: + raise ValueError("Degree must be a positive integer.\n") + + if not isinstance(self.gamma, (str, float)) or ( + isinstance(self.gamma, str) and self.gamma not in self._valid_gammas + ): + raise ValueError("Gamma must be 'scale', 'auto', or a positive float.\n") + + if isinstance(self.gamma, float) and self.gamma <= 0: + raise ValueError("Gamma must be a positive float.\n") + + def _kernel_function(self, X1: np.ndarray, X2: np.ndarray) -> np.ndarray: + """ + Computes the kernel between two sets of data points. + + Args: + X1 (np.ndarray): First set of data points. + X2 (np.ndarray): Second set of data points. + + Returns: + np.ndarray: Kernel matrix. + """ + if self.kernel == "linear": + return linear_kernel(X1, X2) + + if self.kernel == "poly": + return polynomial_kernel(X1, X2, degree=self.degree) + + if self.kernel == "rbf": + return rbf_kernel(X1, X2, gamma=self.gamma) + + def fit(self, X: np.ndarray, y: np.ndarray) -> None: + """ + Fits the SVR model to the training data. + + Args: + X (np.ndarray): Feature matrix. + y (np.ndarray): Target vector. + """ + self._validate_parameters() + + X = convert_array_numpy(X) + y = convert_array_numpy(y) + + self.X_ = X + self.y_ = y + + # Compute the kernel matrix + self.K_ = self._kernel_function(X, X) + + n_samples = X.shape[0] + K = self.K_ + + # Create P matrix + P_top = np.hstack((K, -K)) + P_bottom = np.hstack((-K, K)) + P = np.vstack((P_top, P_bottom)) + + # Ensure P is positive semi-definite + P = P.astype(np.float64) + 1e-8 * np.eye(2 * n_samples) + + # Create q vector + q = np.hstack([self.epsilon - y, self.epsilon + y]) + + # Create G matrix and h vector for inequality constraints + G_std = np.vstack((-np.eye(2 * n_samples), np.eye(2 * n_samples))) + h_std = np.hstack((np.zeros(2 * n_samples), self.C * np.ones(2 * n_samples))) + + # Create A matrix and b vector for equality constraint + A = np.hstack((np.ones(n_samples), -np.ones(n_samples))).reshape(1, -1) + b = np.array([0.0]) + + P = matrix(P) + q = matrix(q) + G = matrix(G_std) + h = matrix(h_std) + A = matrix(A) + b = matrix(b) + + # Solve QP problem + solvers.options["show_progress"] = False # Suppress output + solution = solvers.qp(P, q, G, h, A, b) + + # Extract alphas + z = np.array(solution["x"]).flatten() + alpha = z[:n_samples] + alpha_star = z[n_samples:] + self.alphas_ = alpha - alpha_star + + # Compute bias term + f = self.K_ @ self.alphas_ + idx = np.where((alpha > 1e-5) & (alpha < self.C - 1e-5))[0] + idx_star = np.where((alpha_star > 1e-5) & (alpha_star < self.C - 1e-5))[0] + + b_list = [] + + for i in idx: + b_i = y[i] - f[i] - self.epsilon + b_list.append(b_i) + + for i in idx_star: + b_i = y[i] - f[i] + self.epsilon + b_list.append(b_i) + + if b_list: + self.b_ = np.mean(b_list) + else: + self.b_ = 0.0 # Default to zero if no support vectors found + + def predict(self, X: np.ndarray) -> np.ndarray: + """ + Predicts target values for the given feature matrix. + + Args: + X (np.ndarray): Feature matrix. + + Returns: + np.ndarray: Predicted target values. + """ + if self.X_ is None or self.alphas_ is None: + raise ValueError( + "The model has not been trained yet. Please call the fit method first." + ) + + X = convert_array_numpy(X) + K_pred = self._kernel_function(self.X_, X) + predictions = (self.alphas_ @ K_pred) + self.b_ + return predictions + + def score( + self, X: np.ndarray, y: np.ndarray, metric: str = "r_squared" + ) -> np.float64: + """ + Uses the trained model to predict the classes of a given + data points (also called features). + + Args: + X (np.ndarray): the features array. + y (np.ndarray): the labels array. + metric (string): which metric should be used to assess + the model's performance. Defaults to R Squared. + + Returns: + np.float32: the score achieved by the model. + """ + try: + assert metric in self._valid_score_metrics + except AssertionError as error: + raise ValueError( + f"Invalid value for 'metric'. Must be {self._valid_score_metrics}.\n" + ) from error + + y_hat = self.predict(X) + + if metric == "r_squared": + return r_squared(y, y_hat) + + if metric == "mse": + return mean_squared_error(y, y_hat) + + if metric == "mae": + return mean_absolute_error(y, y_hat) + + if metric == "rmse": + return root_mean_squared_error(y, y_hat) + + if metric == "medae": + return median_absolute_error(y, y_hat) + + if metric == "mape": + return mean_absolute_percentage_error(y, y_hat) + + if metric == "msle": + return mean_squared_logarithmic_error(y, y_hat) + + if metric == "max_error": + return max_error(y, y_hat) diff --git a/scratchml/models/svr.py b/scratchml/models/svr.py deleted file mode 100644 index 624d4d6..0000000 --- a/scratchml/models/svr.py +++ /dev/null @@ -1,234 +0,0 @@ -from abc import ABC -import numpy as np -from ..utils import convert_array_numpy -from cvxopt import matrix, solvers -from ..metrics import ( - mean_squared_error, - root_mean_squared_error, - r_squared, - mean_absolute_error, - median_absolute_error, - mean_absolute_percentage_error, - mean_squared_logarithmic_error, - max_error, -) -from ..kernels import linear_kernel, polynomial_kernel, rbf_kernel - - -class BaseSVR(ABC): - """ - Base class for Support Vector Regression (SVR). - """ - - def __init__( - self, - kernel: str = "rbf", - C: float = 1.0, - epsilon: float = 0.1, - degree: int = 3, - gamma: str = "scale", - ) -> None: - """ - Initializes SVR with default parameters. - - Args: - kernel (str, optional): Kernel type to be used in the algorithm. Defaults to "rbf". - C (float, optional): Regularization parameter. Defaults to 1.0. - epsilon (float, optional): Epsilon parameter in the epsilon-SVR model. Defaults to 0.1. - degree (int, optional): Degree of the polynomial kernel. Ignored by other kernels. Defaults to 3. - gamma (str, optional): Kernel coefficient for ‘rbf’, ‘poly’, and ‘sigmoid’. Defaults to "scale". - """ - self.kernel = kernel - self.C = C - self.epsilon = epsilon - self.degree = degree - self.gamma = gamma - self.X_ = None - self.y_ = None - self.alphas_ = None - self.b_ = None - self.K_ = None - - self._validate_parameters() - - def _validate_parameters(self) -> None: - """ - Validates the parameters passed during initialization. - """ - if not isinstance(self.C, (int, float)) or self.C <= 0: - raise ValueError("C must be a positive number.") - - if self.kernel not in ["linear", "poly", "rbf"]: - raise ValueError("Kernel must be one of 'linear', 'poly', or 'rbf'.") - - if not isinstance(self.epsilon, (int, float)) or self.epsilon < 0: - raise ValueError("Epsilon must be a non-negative number.") - - if not isinstance(self.degree, int) or self.degree <= 0: - raise ValueError("Degree must be a positive integer.") - - if not isinstance(self.gamma, (str, float)) or ( - isinstance(self.gamma, str) and self.gamma not in ["scale", "auto"] - ): - raise ValueError("Gamma must be 'scale', 'auto', or a positive float.") - if isinstance(self.gamma, float) and self.gamma <= 0: - raise ValueError("Gamma must be a positive float.") - - def _kernel_function(self, X1: np.ndarray, X2: np.ndarray) -> np.ndarray: - """ - Computes the kernel between two sets of data points. - - Args: - X1 (np.ndarray): First set of data points. - X2 (np.ndarray): Second set of data points. - - Returns: - np.ndarray: Kernel matrix. - """ - if self.kernel == "linear": - return linear_kernel(X1, X2) - elif self.kernel == "poly": - return polynomial_kernel(X1, X2, degree=self.degree) - elif self.kernel == "rbf": - return rbf_kernel(X1, X2, gamma=self.gamma) - else: - raise ValueError("Unknown kernel specified") - - def fit(self, X: np.ndarray, y: np.ndarray) -> None: - """ - Fits the SVR model to the training data. - - Args: - X (np.ndarray): Feature matrix. - y (np.ndarray): Target vector. - """ - X = convert_array_numpy(X) - y = convert_array_numpy(y) - - self.X_ = X - self.y_ = y - - # Set gamma if necessary - if self.gamma == "scale": - self.gamma_ = 1 / (X.shape[1] * X.var()) - elif self.gamma == "auto": - self.gamma_ = 1 / X.shape[1] - else: - self.gamma_ = float(self.gamma) - - # Compute the kernel matrix - self.K_ = self._kernel_function(X, X) - - n_samples = X.shape[0] - K = self.K_ - - # Create P matrix - P_top = np.hstack((K, -K)) - P_bottom = np.hstack((-K, K)) - P = np.vstack((P_top, P_bottom)) - - # Ensure P is positive semi-definite - P = P.astype(np.float64) + 1e-8 * np.eye(2 * n_samples) - - # Create q vector - q = np.hstack([self.epsilon - y, self.epsilon + y]) - - # Create G matrix and h vector for inequality constraints - G_std = np.vstack((-np.eye(2 * n_samples), np.eye(2 * n_samples))) - h_std = np.hstack((np.zeros(2 * n_samples), self.C * np.ones(2 * n_samples))) - - # Create A matrix and b vector for equality constraint - A = np.hstack((np.ones(n_samples), -np.ones(n_samples))).reshape(1, -1) - b = np.array([0.0]) - - P = matrix(P) - q = matrix(q) - G = matrix(G_std) - h = matrix(h_std) - A = matrix(A) - b = matrix(b) - - # Solve QP problem - solvers.options["show_progress"] = False # Suppress output - solution = solvers.qp(P, q, G, h, A, b) - - # Extract alphas - z = np.array(solution["x"]).flatten() - alpha = z[:n_samples] - alpha_star = z[n_samples:] - self.alphas_ = alpha - alpha_star - - # Compute bias term - f = self.K_ @ self.alphas_ - idx = np.where((alpha > 1e-5) & (alpha < self.C - 1e-5))[0] - idx_star = np.where((alpha_star > 1e-5) & (alpha_star < self.C - 1e-5))[0] - - b_list = [] - - for i in idx: - b_i = y[i] - f[i] - self.epsilon - b_list.append(b_i) - - for i in idx_star: - b_i = y[i] - f[i] + self.epsilon - b_list.append(b_i) - - if b_list: - self.b_ = np.mean(b_list) - else: - self.b_ = 0.0 # Default to zero if no support vectors found - - def predict(self, X: np.ndarray) -> np.ndarray: - """ - Predicts target values for the given feature matrix. - - Args: - X (np.ndarray): Feature matrix. - - Returns: - np.ndarray: Predicted target values. - """ - if self.X_ is None or self.alphas_ is None: - raise ValueError( - "The model has not been trained yet. Please call the fit method first." - ) - - X = convert_array_numpy(X) - K_pred = self._kernel_function(self.X_, X) - predictions = (self.alphas_ @ K_pred) + self.b_ - return predictions - - def score( - self, X: np.ndarray, y: np.ndarray, metric: str = "r_squared" - ) -> np.float64: - """ - Evaluates the model on a test dataset. - - Args: - X (np.ndarray): Feature matrix. - y (np.ndarray): True target values. - metric (str, optional): Evaluation metric. Defaults to "r_squared". - - Returns: - np.float64: Score based on the chosen metric. - """ - y_hat = self.predict(X) - - if metric == "r_squared": - return r_squared(y, y_hat) - elif metric == "mse": - return mean_squared_error(y, y_hat) - elif metric == "mae": - return mean_absolute_error(y, y_hat) - elif metric == "rmse": - return root_mean_squared_error(y, y_hat) - elif metric == "medae": - return median_absolute_error(y, y_hat) - elif metric == "mape": - return mean_absolute_percentage_error(y, y_hat) - elif metric == "msle": - return mean_squared_logarithmic_error(y, y_hat) - elif metric == "max_error": - return max_error(y, y_hat) - else: - raise ValueError(f"Unknown metric: {metric}") diff --git a/tests/models/test_mlp.py b/tests/models/test_mlp.py index 9fcc1c3..5e91f57 100644 --- a/tests/models/test_mlp.py +++ b/tests/models/test_mlp.py @@ -137,7 +137,7 @@ def test_3(self): ) skmlp = SkMLPRegressor( hidden_layer_sizes=(32, 64), - solver="adam", + solver="sgd", early_stopping=False, n_iter_no_change=200, random_state=42, @@ -152,7 +152,7 @@ def test_3(self): score_skmlp = skmlp.score(X, y) score = mlp.score(X, y) - atol = math.floor(y.shape[0] * 0.02) + atol = math.floor(y.shape[0] * 0.07) # Compare model attributes assert mlp.n_features_in_ == skmlp.n_features_in_ diff --git a/tests/models/test_svc.py b/tests/models/test_svm.py similarity index 56% rename from tests/models/test_svc.py rename to tests/models/test_svm.py index 751c0f0..395ed9a 100644 --- a/tests/models/test_svc.py +++ b/tests/models/test_svm.py @@ -1,17 +1,18 @@ import math from numpy.testing import assert_allclose, assert_equal from sklearn.svm import SVC as SkSVC -from scratchml.models.svc import SVC +from sklearn.svm import SVR as SkSVR +from scratchml.models.svm import SVC, SVR from scratchml.scalers import StandardScaler -from ..utils import generate_classification_dataset, repeat +from ..utils import generate_classification_dataset, repeat, generate_regression_dataset import unittest import numpy as np from numpy.testing import assert_array_equal -class Test_SVC(unittest.TestCase): +class Test_SVM(unittest.TestCase): """ - Unit test class for the custom SVC implementation. + Unit test class for the custom SVM implementation. """ @repeat(3) @@ -27,7 +28,7 @@ def test_binary_classification(self): custom_svc = SVC(kernel="linear") sklearn_svc = SkSVC(kernel="linear", max_iter=1000) - custom_svc.fit(X, y ) + custom_svc.fit(X, y) sklearn_svc.fit(X, y) # Predict and score @@ -171,6 +172,134 @@ def test_model_parameters(self): "Support vectors should match between implementations.", ) + @repeat(3) + def test_linear_kernel(self): + """ + Test the custom SVR with linear kernel against Scikit-Learn's SVR. + """ + X, y = generate_regression_dataset(n_samples=1000, n_features=3) + + custom_svr = SVR(kernel="linear", C=1.0, epsilon=0.1) + sklearn_svr = SkSVR(kernel="linear", C=1.0, epsilon=0.1) + + custom_svr.fit(X, y) + sklearn_svr.fit(X, y) + + custom_pred = custom_svr.predict(X) + sklearn_pred = sklearn_svr.predict(X) + + custom_score = custom_svr.score(X, y) + sklearn_score = sklearn_svr.score(X, y) + + atol = 1e-1 + assert_allclose(custom_pred, sklearn_pred, atol=atol, rtol=1e-2) + assert abs(custom_score - sklearn_score) / abs(sklearn_score) < 0.1 + + def test_untrained_model_prediction_error_svr(self): + """ + Ensure an error is raised when predicting with an untrained model. + """ + svr = SVR(kernel="linear") + X, _ = generate_regression_dataset(n_samples=10, n_features=2) + + with self.assertRaises(ValueError): + svr.predict(X) + + @repeat(3) + def test_custom_kernel_initialization_svr(self): + """ + Ensure the SVR model initializes correctly with a custom kernel. + """ + svr = SVR(kernel="poly") + self.assertEqual( + svr.kernel, + "poly", + "Model should initialize with 'poly' kernel.", + ) + + @repeat(3) + def test_output_type_and_shape_svr(self): + """ + Validate that the output type and shape of predictions are correct. + """ + X, y = generate_regression_dataset(n_samples=200, n_features=5) + + custom_svr = SVR(kernel="linear") + sklearn_svr = SkSVR(kernel="linear") + + custom_svr.fit(X, y) + sklearn_svr.fit(X, y) + + custom_pred = custom_svr.predict(X) + sklearn_pred = sklearn_svr.predict(X) + + self.assertIsInstance(custom_pred, np.ndarray) + self.assertEqual(custom_pred.shape, sklearn_pred.shape) + + @repeat(3) + def test_model_score_metrics(self): + """ + Compare the model scores using different metrics. + """ + # Generate dataset and ensure non-negative targets + X, y = generate_regression_dataset(n_samples=200, n_features=5) + + # Make y non-negative for MSLE compatibility + y = np.abs(y) # Ensure non-negativity + + custom_svr = SVR(kernel="linear") + custom_svr.fit(X, y) + + metrics = [ + "r_squared", + "mse", + "mae", + "rmse", + "medae", + "mape", + "msle", # This is the problematic metric + "max_error", + ] + + for metric in metrics: + score = custom_svr.score(X, y, metric=metric) + self.assertTrue( + isinstance(score, (float, np.float64)), + f"Score for metric {metric} should be a float.", + ) + self.assertFalse(np.isnan(score), f"Score for metric {metric} is NaN.") + + def test_parameter_validation(self): + """ + Test parameter validation for the custom SVR implementation. + """ + # Generate dataset and ensure non-negative targets + X, y = generate_regression_dataset(n_samples=200, n_features=5) + + with self.assertRaises(ValueError): + svr = SVR(kernel="invalid_kernel") + svr.fit(X, y) + + with self.assertRaises(ValueError): + svr = SVR(C=-1.0) + svr.fit(X, y) + + with self.assertRaises(ValueError): + svr = SVR(epsilon=-0.1) + svr.fit(X, y) + + with self.assertRaises(ValueError): + svr = SVR(degree=-1) + svr.fit(X, y) + + with self.assertRaises(ValueError): + svr = SVR(gamma="invalid_gamma") + svr.fit(X, y) + + with self.assertRaises(ValueError): + svr = SVR(gamma=-1.0) + svr.fit(X, y) + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tests/models/test_svr.py b/tests/models/test_svr.py deleted file mode 100644 index 181e5ac..0000000 --- a/tests/models/test_svr.py +++ /dev/null @@ -1,135 +0,0 @@ -import unittest -import numpy as np -from numpy.testing import assert_allclose, assert_array_equal -from scratchml.models.svr import BaseSVR -from sklearn.svm import SVR as SkSVR -from ..utils import generate_regression_dataset, repeat - - -class Test_SVR(unittest.TestCase): - """ - Unit test class for the custom SVR implementation. - """ - - @repeat(3) - def test_linear_kernel(self): - """ - Test the custom SVR with linear kernel against Scikit-Learn's SVR. - """ - X, y = generate_regression_dataset(n_samples=1000, n_features=3) - - custom_svr = BaseSVR(kernel="linear", C=1.0, epsilon=0.1) - sklearn_svr = SkSVR(kernel="linear", C=1.0, epsilon=0.1) - - custom_svr.fit(X, y) - sklearn_svr.fit(X, y) - - custom_pred = custom_svr.predict(X) - sklearn_pred = sklearn_svr.predict(X) - - custom_score = custom_svr.score(X, y) - sklearn_score = sklearn_svr.score(X, y) - - atol = 1e-1 - assert_allclose(custom_pred, sklearn_pred, atol=atol, rtol=1e-2) - assert abs(custom_score - sklearn_score) / abs(sklearn_score) < 0.1 - - def test_untrained_model_prediction_error(self): - """ - Ensure an error is raised when predicting with an untrained model. - """ - svr = BaseSVR(kernel="linear") - X, _ = generate_regression_dataset(n_samples=10, n_features=2) - - with self.assertRaises(ValueError): - svr.predict(X) - - @repeat(3) - def test_custom_kernel_initialization(self): - """ - Ensure the SVR model initializes correctly with a custom kernel. - """ - svr = BaseSVR(kernel="poly") - self.assertEqual( - svr.kernel, - "poly", - "Model should initialize with 'poly' kernel.", - ) - - @repeat(3) - def test_output_type_and_shape(self): - """ - Validate that the output type and shape of predictions are correct. - """ - X, y = generate_regression_dataset(n_samples=200, n_features=5) - - custom_svr = BaseSVR(kernel="linear") - sklearn_svr = SkSVR(kernel="linear") - - custom_svr.fit(X, y) - sklearn_svr.fit(X, y) - - custom_pred = custom_svr.predict(X) - sklearn_pred = sklearn_svr.predict(X) - - self.assertIsInstance(custom_pred, np.ndarray) - self.assertEqual(custom_pred.shape, sklearn_pred.shape) - - @repeat(3) - def test_model_score_metrics(self): - """ - Compare the model scores using different metrics. - """ - # Generate dataset and ensure non-negative targets - X, y = generate_regression_dataset(n_samples=200, n_features=5) - - # Make y non-negative for MSLE compatibility - y = np.abs(y) # Ensure non-negativity - - custom_svr = BaseSVR(kernel="linear") - custom_svr.fit(X, y) - - metrics = [ - "r_squared", - "mse", - "mae", - "rmse", - "medae", - "mape", - "msle", # This is the problematic metric - "max_error", - ] - - for metric in metrics: - score = custom_svr.score(X, y, metric=metric) - self.assertTrue( - isinstance(score, (float, np.float64)), - f"Score for metric {metric} should be a float.", - ) - self.assertFalse(np.isnan(score), f"Score for metric {metric} is NaN.") - - def test_parameter_validation(self): - """ - Test parameter validation for the custom SVR implementation. - """ - with self.assertRaises(ValueError): - BaseSVR(kernel="invalid_kernel") - - with self.assertRaises(ValueError): - BaseSVR(C=-1.0) - - with self.assertRaises(ValueError): - BaseSVR(epsilon=-0.1) - - with self.assertRaises(ValueError): - BaseSVR(degree=-1) - - with self.assertRaises(ValueError): - BaseSVR(gamma="invalid_gamma") - - with self.assertRaises(ValueError): - BaseSVR(gamma=-1.0) - - -if __name__ == "__main__": - unittest.main(verbosity=2) From 17a74743155cde1e7a55b948b63715a735b5cf07 Mon Sep 17 00:00:00 2001 From: Rafael Greca Date: Tue, 3 Dec 2024 19:40:20 -0300 Subject: [PATCH 3/4] fixing pylint issues and updating workflows --- .github/workflows/pylint.yml | 39 +++++++++++++++++++++-------------- .github/workflows/test.yml | 27 ++++++++++++++++++++++++ .github/workflows/unitest.yml | 24 --------------------- scratchml/kernels.py | 8 +++---- scratchml/models/svm.py | 4 ++-- 5 files changed, 56 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/test.yml delete mode 100644 .github/workflows/unitest.yml diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index d143d4d..18bfac0 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,24 +1,31 @@ name: Pylint -on: [push] +on: + push: + branches: ["main", "develop"] + pull_request: + branches: ["main", "develop"] jobs: build: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10"] steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pylint - - name: Analysing the code with pylint - run: | - pylint $(git ls-files '*.py') --rcfile=.pylintrc + - uses: actions/checkout@v4 + - name: Set up Python "3.11" + uses: actions/setup-python@v3 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint black + + - name: Run Black + run: | + black . --safe + + - name: Analysing the code with pylint + run: | + pylint $(git ls-files '*.py') --rcfile=.pylintrc diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..80c788d --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,27 @@ +name: Test + +on: + push: + branches: ["main", "develop"] + pull_request: + branches: ["main", "develop"] + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python "3.11" + uses: actions/setup-python@v3 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements/requirements_test.txt ]; then pip install -r requirements/requirements_test.txt; fi + + - name: Test with unittest + run: | + python3 -m unittest discover -p 'test_*.py' diff --git a/.github/workflows/unitest.yml b/.github/workflows/unitest.yml deleted file mode 100644 index c21c847..0000000 --- a/.github/workflows/unitest.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: ci-python-unittest - -on: [push] - -jobs: - - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10"] - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r ./requirements/requirements_test.txt - - name: Test with unittest - run: | - python3 -m unittest discover -p 'test_*.py' diff --git a/scratchml/kernels.py b/scratchml/kernels.py index 1031880..4fd20e5 100644 --- a/scratchml/kernels.py +++ b/scratchml/kernels.py @@ -45,11 +45,11 @@ def rbf_kernel(X1: np.ndarray, X2: np.ndarray, gamma: str = "scale") -> np.ndarr Returns: np.ndarray: RBF kernel matrix. """ + _gamma = 1.0 + if gamma == "scale": - gamma = 1.0 / X1.shape[1] - elif gamma == "auto": - gamma = 1.0 + _gamma = 1.0 / X1.shape[1] X1_norm = np.sum(X1**2, axis=1).reshape(-1, 1) X2_norm = np.sum(X2**2, axis=1).reshape(1, -1) - return np.exp(-gamma * (X1_norm + X2_norm - 2 * np.dot(X1, X2.T))) + return np.exp((-1 * _gamma) * (X1_norm + X2_norm - 2 * np.dot(X1, X2.T))) diff --git a/scratchml/models/svm.py b/scratchml/models/svm.py index 3ae1b53..c81fee2 100644 --- a/scratchml/models/svm.py +++ b/scratchml/models/svm.py @@ -632,8 +632,8 @@ def fit(self, X: np.ndarray, y: np.ndarray) -> None: K = self.K_ # Create P matrix - P_top = np.hstack((K, -K)) - P_bottom = np.hstack((-K, K)) + P_top = np.hstack((K, -1 * K)) + P_bottom = np.hstack((-1 * K, K)) P = np.vstack((P_top, P_bottom)) # Ensure P is positive semi-definite From 0238ff27c949d2dd0562be452ff0fb136f12c7fc Mon Sep 17 00:00:00 2001 From: Rafael Greca Date: Tue, 3 Dec 2024 19:45:57 -0300 Subject: [PATCH 4/4] fixing binary labeling issue in perceptron model --- docs/ROADMAP.md | 2 +- scratchml/models/perceptron.py | 6 +++++ tests/models/test_perceptron.py | 46 +++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index da801e4..af3d6a9 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -87,7 +87,7 @@ ## Known Issues -- [ ] Binary Labeling Issue in Perceptron Model +- [x] Binary Labeling Issue in Perceptron Model - [ ] CI Unit test workflow not working (Numpy version error) - [ ] MLP not working properly when dealing with a multiclass classification problem (I think it's a vanishing gradient problem) - [ ] Sometimes the Logistic Regression model gets stuck (I think it's a vanishing gradient problem) diff --git a/scratchml/models/perceptron.py b/scratchml/models/perceptron.py index c84f150..4ee99ad 100644 --- a/scratchml/models/perceptron.py +++ b/scratchml/models/perceptron.py @@ -80,6 +80,12 @@ def fit(self, X: np.ndarray, y: np.ndarray) -> None: y = convert_array_numpy(y).reshape(-1, 1) if len(self.classes_) == 2: + if -1 in self.classes_: + print("WARNING: Changing the -1 class label to 0.\n") + + # Map the unique classes to 0 and 1 + y = np.where(y == self.classes_[0], 0, 1) + self.intercept_ = np.zeros((1,), dtype=np.float64) self.coef_ = np.zeros((X.shape[1], 1), dtype=np.float64) self.coef_, self.intercept_ = self._fitting_model( diff --git a/tests/models/test_perceptron.py b/tests/models/test_perceptron.py index 15e6376..de1cce8 100644 --- a/tests/models/test_perceptron.py +++ b/tests/models/test_perceptron.py @@ -540,6 +540,52 @@ def test_12(self): assert_equal(predict_skp.shape, predict_p.shape) assert_allclose(predict_skp, predict_p, atol=atol) + @repeat(3) + def test_13(self): + """ + Test the Perceptron implementation with classes -1 and 1, then + compares it to the Scikit-Learn implementation. + """ + X, y = generate_classification_dataset( + n_samples=5000, n_features=2, n_classes=2, n_clusters_per_class=1 + ) + y = np.where(y == 0, -1, 1) + + perceptron = Perceptron( + penalty=None, + lr=0.001, + alpha=0.0001, + fit_intercept=True, + max_iter=1000, + tol=0.001, + verbose=0, + n_jobs=None, + ) + skperceptron = SkPerceptron( + penalty=None, + alpha=0.0001, + fit_intercept=True, + max_iter=1000, + tol=0.001, + verbose=0, + n_jobs=None, + ) + + skperceptron.fit(X, y) + perceptron.fit(X, y) + + predict_skp = skperceptron.predict(X) + predict_p = np.squeeze(perceptron.predict(X)) + + atol = math.floor(y.shape[0] * 0.05) + + assert_equal(skperceptron.coef_.shape, perceptron.coef_.reshape(1, -1).shape) + assert_equal(skperceptron.intercept_.shape, perceptron.intercept_.shape) + assert_equal(skperceptron.n_features_in_, perceptron.n_features_in_) + # assert_equal(skperceptron.classes_, perceptron.classes_) + assert_equal(predict_skp.shape, predict_p.shape) + assert_allclose(predict_skp, predict_p, atol=atol) + if __name__ == "__main__": unittest.main(verbosity=2)