Merge pull request #43 from rajadilipkolli/main

Fix Edge cases, adds workflows and updates dependencies

.devcontainer/devcontainer.json

@@ -0,0 +1,35 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
+{
+	"name": "Python 3",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/python:1-3.12-bullseye",
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.python",
+				"Vue.volar",
+				"ms-python.debugpy"
+			]
+		}
+	},
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	"features": {
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
+		"ghcr.io/devcontainers/features/github-cli:1": {},
+		"ghcr.io/devcontainers/features/node:1": {}
+	},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [5050, 8000]
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "pip3 install --user -r requirements.txt",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.github/dependabot.yml

@@ -0,0 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for more information:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+# https://containers.dev/guide/dependabot
+
+version: 2
+updates:
+ - package-ecosystem: "devcontainers"
+   directory: "/"
+   schedule:
+     interval: weekly

.github/workflows/api.yml

@@ -0,0 +1,51 @@
+name: Django Tests CI
+
+on:
+  push:
+    paths:
+      - "api/**"
+    branches: [ main ]
+  pull_request:
+    paths:
+      - "api/**"
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+jobs:
+  build:
+    name: Run Unit & Integration Tests
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: "./api"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      - name: Install Required softwares
+        run: |
+          cd ..
+          docker compose up -d timedb cache
+          sleep 5
+          docker ps -a
+      - name: Test
+        env:
+          SECRET_KEY: '%k8!x4pm=mf!iqz^jws)ijn=)-md-uf_i=^mya1t*d!4f#^74k'
+          DATABASE_URL: psql://postgres:foliop4sswd@127.0.0.1:15432/folioman
+          CACHE_URL: rediscache://localhost:16379/1?client_class=django_redis.client.DefaultClient&timeout=86400
+          CELERY_BROKER_URL: redis://localhost:16379/3
+          CELERY_RESULT_BACKEND: redis://localhost:16379/5
+          QUANDL_API_KEY: PUT_QUANDL_API_HERE
+          ENVIRONMENT: dev
+        run: python manage.py test folioman.tests mutualfunds.tests

.github/workflows/ui.yml

@@ -0,0 +1,36 @@
+name: Nuxt Build CI
+
+on:
+  push:
+    paths:
+      - "ui/**"
+    branches: [ main ]
+  pull_request:
+    paths:
+      - "ui/**"
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+jobs:
+  build:
+    name: Run Unit & Integration Tests
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: "./ui"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+      - name: Install dependencies
+        run: yarn
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "16"
+      - name: RUN build
+        run: |
+          yarn install
+          yarn run build

.gitpod.yml

@@ -0,0 +1,5 @@
+
+vscode:
+  extensions:
+    - ms-python.python
+    - Vue.volar

.vscode/launch.json

@@ -0,0 +1,42 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Django",
+            "type": "debugpy",
+            "request": "launch",
+            "args": [
+                "runserver"
+            ],
+            "django": true,
+            "autoStartBrowser": false,
+            "program": "${workspaceFolder}/api/manage.py"
+        },
+        {
+            "name": "Celery",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "celery",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/api",
+            "args": [
+                "-A",
+                "taskman",
+                "worker",
+                "-l",
+                "info",
+                "-P",
+                "solo",
+            ]
+        }
+    ],
+    "compounds": [
+        {
+            "name": "Celery and Django",
+            "configurations": ["Celery", "Django"]
+        }
+    ]
+}

README.md

@@ -22,6 +22,7 @@ Once the process is complete, visit the following urls to get started.
 
 - http://localhost:8000 - frontend
 - http://localhost:8000/admin/ - backend
+- http://localhost:5050 - pgadmin
 
 The default username and password is given below; it can be changed from the backend dashboard. 
 ```

api/ReadMe.md

@@ -0,0 +1,33 @@
+
+## Start in local 
+
+assuming python is installed, open code in Visual Studio Code, right-click and select 'Run Python'.
+
+This will prompt you to create a virtual environment, select `.venv`, and install all required software.
+
+then open the terminal and issue the command below to set up lookup data
+
+```shell 
+python manage.py migrate
+```
+
+Issue below command to start local server
+
+```shell
+python manage.py runserver
+```
+
+## How to update the dependencies in virtual environment
+
+```shell
+pip install -r requirements.txt
+```
+
+## Running the Celery worker
+In the new terminal tab, run the following command:
+
+```shell
+celery -A taskman worker -l info -P solo
+```
+
+where celery is the version of Celery, with the -A option to specify the celery instance to use (in our case, it's celery in the app.py file, so it's taskman), and worker is the subcommand to run the worker, and --loglevel=info to set the verbosity log level to INFO.

api/env.example

@@ -5,8 +5,6 @@ DATABASE_URL=psql://postgres:foliop4sswd@timedb:5432/folioman
 CACHE_URL=rediscache://cache:6379/1?client_class=django_redis.client.DefaultClient&timeout=86400
 CELERY_BROKER_URL=redis://cache:6379/3
 CELERY_RESULT_BACKEND=redis://cache:6379/5
-QUANDL_API_KEY=PUT_QUANDL_API_HERE
-# help: https://help.quandl.com/article/320-where-can-i-find-my-api-key
 
 # Uncomment and update the following variable to enable sentry logging support.
 # SENTRY_DSN="https://projectcode@sentry.example.com/pid"

api/folioman/urls.py

@@ -1,7 +1,7 @@
 from django.urls import path
 from rest_framework.urlpatterns import format_suffix_patterns
-from . import views
 
+from . import views
 
 urlpatterns = [
     path("me", views.UserView.as_view(), name="me"),

api/mutualfunds/importers/cas.py

@@ -1,9 +1,8 @@
-from datetime import date
 import re
+from typing import List
 
 from casparser.types import CASParserDataType, FolioType
 from dateutil.parser import parse as dateparse
-from typing import List
 
 from mutualfunds.models import (
     Portfolio,
@@ -26,19 +25,19 @@ def import_cas(data: CASParserDataType, user_id):
     if not (email and name):
         raise ValueError("Email or Name invalid!")
 
+    folios: List[FolioType] = data.get("folios", []) or []
     try:
         pf = Portfolio.objects.get(email=email)
     except Portfolio.DoesNotExist:
         pf = Portfolio(
-            email=email, name=name, user_id=user_id, pan=(investor_info.get("pan") or "").strip()
+            email=email, name=name, user_id=user_id, pan=(folios[0].get("PAN") or "").strip()
         )
         pf.save()
 
     num_created = 0
     num_total = 0
     new_folios = 0
 
-    folios: List[FolioType] = data.get("folios", []) or []
     fund_scheme_ids = []
     scheme_dates = {}
     for folio in folios:
@@ -92,6 +91,8 @@ def import_cas(data: CASParserDataType, user_id):
                 folio["PANKYC"] = "notok"
             if not folio["PAN"]:
                 folio["PAN"] = "noregister"
+            if not folio["KYC"]:
+                folio["KYC"] = "notok"
             folio_obj = Folio(
                 amc_id=folio["amc_id"],
                 number=folio_number,
@@ -131,7 +132,7 @@ def import_cas(data: CASParserDataType, user_id):
                     units=str(transaction["units"] or 0),
                     defaults={
                         "description": transaction["description"].strip(),
-                        "amount": transaction["amount"],
+                        "amount": transaction["amount"] or 0.00001,
                         "nav": transaction["nav"] or 0,
                         "order_type": Transaction.get_order_type(
                             transaction["description"], transaction["amount"]

api/mutualfunds/importers/daily_value.py

@@ -1,10 +1,10 @@
 """Bulk import various DailyValue models"""
 
+import pandas as pd
 from import_export.fields import Field
 from import_export.instance_loaders import ModelInstanceLoader
 from import_export.resources import ModelResource
 from import_export.widgets import DateWidget
-import pandas as pd
 
 from mutualfunds.models import FolioValue, PortfolioValue, SchemeValue
 

api/mutualfunds/importers/fetcher.py

@@ -2,11 +2,10 @@
 import io
 import logging
 import re
-import zipfile
 
+import requests
 from django.conf import settings
 from lxml.html import fromstring
-import requests
 from requests.utils import default_user_agent
 
 logger = logging.getLogger(__name__)
@@ -35,49 +34,6 @@ def fetch_bse_star_master_data():
     logger.info("BSE Master data downloaded.")
     return response.text
 
-
-def fetch_quandl_amfi_metadata():
-    logger.info("Downloading quandl AMFI data...")
-    params = {"api_key": settings.QUANDL_API_KEY}
-    response = requests.get(settings.QUANDL_METADATA_URL, params=params, timeout=300)
-    if response.status_code != 200:
-        raise requests.exceptions.RequestException("Invalid response!")
-    content = io.BytesIO(response.content)
-    data = {}
-    dividends = ("payout", "reinvest")
-
-    def category(x):
-        name = x["name"]
-        desc = x["description"].lower()
-        if "payout" in name.lower() or (
-            "payout" in desc and "reinvest" not in desc and "growth" not in desc
-        ):
-            return "payout"
-        elif "reinvest" in name.lower() or (
-            "payout" not in desc and "reinvest" in desc and "growth" not in desc
-        ):
-            return "reinvest"
-        return "n/a"
-
-    with zipfile.ZipFile(content) as zipf:
-        with zipf.open("AMFI_metadata.csv") as csvf:
-            reader = csv.DictReader(io.TextIOWrapper(csvf, encoding="utf-8"))
-            for row in reader:
-                isins = re.findall(r"\sIN[a-zA-Z0-9]{10}", row["description"])
-
-                is_dividend = False
-                if len(isins) > 2:
-                    raise ValueError("Unexpected layout. Debug needed")
-                elif len(isins) == 2:
-                    is_dividend = True
-                for idx, isin in enumerate(isins):
-                    if is_dividend:
-                        data[isin.strip()] = row, dividends[idx]
-                    else:
-                        data[isin.strip()] = row, category(row)
-    return data
-
-
 def fetch_amfi_scheme_data():
     logger.info("Downloading AMFI scheme data...")
     response = requests.get(settings.AMFI_SCHEME_DATA_URL, timeout=300)

api/mutualfunds/importers/master.py

@@ -7,17 +7,17 @@
 import tablib
 from dateutil.parser import parse as dateparse
 from import_export.fields import Field
-from import_export.resources import ModelResource
 from import_export.instance_loaders import CachedInstanceLoader
+from import_export.resources import ModelResource
 from import_export.results import Error, Result, RowResult
 from rapidfuzz import fuzz, process
 
-from mutualfunds.models import AMC, FundCategory, FundScheme
 from mutualfunds.importers.fetcher import (
     fetch_amfi_code_isin_mapping,
     fetch_amfi_scheme_data,
     fetch_bse_star_master_data
 )
+from mutualfunds.models import AMC, FundCategory, FundScheme
 
 logger = logging.getLogger(__name__)