init

.github/workflows/build_and_release.yaml

@@ -0,0 +1,52 @@
+name: Build and Upload to PyPI
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-and-upload:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel build toml ruff
+
+    - name: Run lint
+      run: |
+        ruff check     
+
+    - name: Run tests
+      run: |
+        pip install pytest
+        pytest -vvvv
+
+    - name: Extract version from pyproject.toml
+      id: get_version
+      run: |
+        echo "VERSION=$(python -c 'import toml; pyproject = toml.load("pyproject.toml"); print(pyproject["project"]["version"])')" >> $GITHUB_ENV
+            
+    - name: Build package
+      run: python -m build --wheel
+
+    - name: Create a Release
+      uses: elgohr/Github-Release-Action@v5
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        title: Release v${{ env.VERSION }}
+
+    - name: Publish to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}
+        packages_dir: dist
+        package_name: msfabricutils
+        version: ${{ env.VERSION }}

.github/workflows/ci.yaml

@@ -0,0 +1,31 @@
+name: Continuous Integration
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12']
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        # pip install setuptools wheel ruff
+    - name: Run lint
+      run: |
+        ruff check
+    - name: Run tests
+      run: |
+        pip install pytest
+        pytest

.github/workflows/docs.yml

@@ -0,0 +1,20 @@
+name: docs
+
+on:
+  workflow_dispatch:  # Allows manual triggering of the workflow
+
+permissions:
+  contents: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.x
+      - run: |
+          python -m pip install --upgrade pip
+          pip install -e .[docs]
+      - run: mkdocs gh-deploy --force

.gitignore

@@ -0,0 +1,135 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include the Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv’s lock file may be unsuitable for version control.
+Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# Ruff
+.ruff_cache

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) [2024] [Jimmy Jensen]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,68 @@
+# MSFabricUtils
+A collection of Spark-free Python utilities for working with Microsoft Fabric.
+## Features
+### DuckDB Connection Wrapper
+Seamless integration between DuckDB and Microsoft Fabric Lakehouses
+Cross-workspace and cross-lakehouse querying capabilities
+Automatic table registration and authentication
+Support for Delta Lake tables
+Flexible table name referencing (1-part to 4-part names)
+## Installation
+```bash
+pip install msfabricutils
+```
+## Quick Start
+```python
+from msfabricutils import FabricDuckDBConnection
+# Initialize connection
+access_token = notebookutils.credentials.getToken('storage')
+conn = FabricDuckDBConnection(access_token=access_token)
+
+# Register lakehouses from different workspaces
+conn.register_workspace_lakehouses(
+    workspace_id='12345678-1234-5678-1234-567812345678', # Sales workspace
+    lakehouses=['sales']
+)
+
+conn.register_workspace_lakehouses(
+    workspace_id='87654321-8765-4321-8765-432187654321', # Marketing workspace
+    lakehouses=['marketing']
+)
+
+# Query across workspaces using fully qualified names
+df = conn.sql("""
+    SELECT
+        c.customer_id,
+        c.name,
+        c.region,
+        s.segment,
+        s.lifetime_value
+    FROM sales_workspace.sales.main.customers c
+    JOIN marketing_workspace.marketing.main.customer_segments s
+    ON c.customer_id = s.customer_id
+    WHERE c.region = 'EMEA'
+""").df()
+
+# Write results back to a lakehouse
+conn.write(df, 'sales.customer_segments_emea', workspace_id='12345678-1234-5678-1234-567812345678')
+```
+
+## Documentation
+For detailed documentation, examples, and API reference, visit our [GitHub Pages documentation](https://mrjsj.github.io/msfabricutils/).
+
+## Table Name Formats
+The DuckDB Connection wrapper supports multiple table name formats (if unambiguous):
+- 4-part: workspace.lakehouse.schema.table
+- 3-part: lakehouse.schema.table
+- 2-part: lakehouse.table
+- 1-part: table
+
+# Contributing
+Contributions are welcome! Here are some ways you can contribute:
+Report bugs and feature requests through GitHub issues
+Submit pull requests for bug fixes or new features
+Improve documentation
+Share ideas for new utilities
+
+# License
+This project is licensed under the MIT License - see the LICENSE file for details.

docs/gen_ref_pages.py

@@ -0,0 +1,36 @@
+"""Generate the code reference pages."""
+
+from pathlib import Path
+import mkdocs_gen_files
+
+nav = mkdocs_gen_files.Nav()
+
+
+# Iterate through all Python files in src directory
+for path in sorted(Path("src").rglob("*.py")):
+    module_path = path.relative_to("src").with_suffix("")
+    doc_path = path.relative_to("src").with_suffix(".md")
+    full_doc_path = Path("reference", doc_path)
+
+    parts = tuple(module_path.parts)
+
+    if parts[-1] == "__init__":
+        continue
+        parts = parts[:-1]
+        doc_path = doc_path.with_name("index.md")
+        full_doc_path = full_doc_path.with_name("index.md")
+    elif parts[-1] == "__main__":
+        continue
+
+    nav[parts] = doc_path.as_posix()
+
+    with mkdocs_gen_files.open(full_doc_path, "w") as fd:
+        identifier = ".".join(parts)
+        #print(f"# {identifier}", file=fd)
+        print("::: " + identifier, file=fd)
+
+    mkdocs_gen_files.set_edit_path(full_doc_path, path)
+
+# Generate navigation summary
+with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file:
+    nav_file.writelines(nav.build_literate_nav())