Merge pull request #175 from almenscorner/dev

v2.1.2

codecov.yml

@@ -9,5 +9,6 @@ ignore:
   - "src/IntuneCD/document_entra.py"
   - "src/IntuneCD/run_documentation.py"
   - "src/IntuneCD/intunecdlib/get_accesstoken.py"
+  - "src/IntuneCD/intunecdlib/logger.py"
   - "src/IntuneCD/backup/Intune/backup_autopilotDevices.py"
   - "src/IntuneCD/backup/Intune/backup_activationLock.py"

index.html

@@ -152,7 +152,7 @@ <h2 class="mb-4">Get started now!</h2>
                         </div>
                     </div>
                     <div class="col-xl-3 mb-4">
-                        <h2 class="mb-4">Follow on Twitter!</h2>
+                        <h2 class="mb-4">Follow on X!</h2>
                         <div class="col-auto">
                             <a class="btn btn-primary btn-lg" href="https://twitter.com/intunecd">Follow</a>
                         </div>
@@ -182,7 +182,7 @@ <h2 class="mb-4">Join IntuneCD Slack!</h2>
                             <li class="list-inline-item">⋅</li>
                             <li class="list-inline-item"><a href="#!" class="text-primary">Contact</a></li>
                         </ul>--->
-                        <p class="text-muted small mb-4 mb-lg-0">&copy; IntuneCD 2023. All Rights Reserved.</p>
+                        <p class="text-muted small mb-4 mb-lg-0">&copy; IntuneCD 2024. All Rights Reserved.</p>
                     </div>
                     <div class="col-lg-6 h-100 text-center text-lg-end my-auto">
                         <ul class="list-inline mb-0">

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = IntuneCD
-version = 2.1.1
+version = 2.1.2
 author = Tobias Almén
 author_email = almenscorner@outlook.com
 description = Tool to backup and update configurations in Intune

src/IntuneCD/backup/Intune/backup_compliancePartner.py

@@ -17,7 +17,7 @@
 
 
 # Get all Compliance Partners and save them in specified path
-def savebackup(path, output, token, append_id):
+def savebackup(path, output, exclude, token, append_id):
     """
     Saves all Compliance Partners in Intune to a JSON or YAML file.
 
@@ -44,6 +44,12 @@ def savebackup(path, output, token, append_id):
         fname = clean_filename(partner["displayName"])
         if append_id:
             fname = f"{fname}__{graph_id}"
+
+        if (
+            partner.get("lastHeartbeatDateTime")
+            and "CompliancePartnerHeartbeat" in exclude
+        ):
+            partner.pop("lastHeartbeatDateTime", None)
         # Save Compliance policy as JSON or YAML depending on configured
         # value in "-o"
         save_output(output, configpath, fname, partner)

src/IntuneCD/backup/Intune/backup_profiles.py

@@ -112,6 +112,8 @@ def savebackup(path, output, exclude, token, prefix, append_id, ignore_omasettin
                         )
                         decoded_oma["value"] = oma_value
                         omas.append(decoded_oma)
+                    else:
+                        omas.append(setting)
 
             else:
                 omas = profile["omaSettings"]

src/IntuneCD/backup_intune.py

@@ -111,7 +111,7 @@ def backup_intune(results, path, output, exclude, token, prefix, append_id, args
     if "CompliancePartner" not in exclude:
         from .backup.Intune.backup_compliancePartner import savebackup
 
-        results.append(savebackup(path, output, token, append_id))
+        results.append(savebackup(path, output, exclude, token, append_id))
 
     if "ManagementPartner" not in exclude:
         from .backup.Intune.backup_managementPartner import savebackup

src/IntuneCD/intunecdlib/get_accesstoken.py

@@ -91,7 +91,7 @@ def obtain_accesstoken_cert(TENANT_NAME, CLIENT_ID, THUMBPRINT, KEY_FILE):
     return token
 
 
-def obtain_accesstoken_interactive(TENANT_NAME, CLIENT_ID):
+def obtain_accesstoken_interactive(TENANT_NAME, CLIENT_ID, scopes):
     """
     This function is used to get an access token to MS Graph interactivly.
 
@@ -109,17 +109,6 @@ def obtain_accesstoken_interactive(TENANT_NAME, CLIENT_ID):
 
     token = None
 
-    # Set the requited scopes
-    scopes = [
-        "DeviceManagementApps.ReadWrite.All",
-        "DeviceManagementConfiguration.ReadWrite.All",
-        "DeviceManagementManagedDevices.Read.All",
-        "DeviceManagementServiceConfig.ReadWrite.All",
-        "Group.Read.All",
-        "Policy.ReadWrite.ConditionalAccess",
-        "Policy.Read.All",
-    ]
-
     try:
         # Get the token interactively
         token = app.acquire_token_interactive(

src/IntuneCD/intunecdlib/get_authparams.py

@@ -15,7 +15,7 @@
 )
 
 
-def getAuth(mode, localauth, certauth, interactiveauth, entra, tenant):
+def getAuth(mode, localauth, certauth, interactiveauth, scopes, entra, tenant):
     """
     This function authenticates to MS Graph and returns the access token.
 
@@ -42,7 +42,7 @@ def getAuth(mode, localauth, certauth, interactiveauth, entra, tenant):
         if not all([TENANT_NAME, CLIENT_ID]):
             raise ValueError("One or more os.environ variables not set")
 
-        return obtain_accesstoken_interactive(TENANT_NAME, CLIENT_ID)
+        return obtain_accesstoken_interactive(TENANT_NAME, CLIENT_ID, scopes)
 
     if mode:
         if mode == "devtoprod":

src/IntuneCD/intunecdlib/graph_batch.py

@@ -11,6 +11,7 @@
 import time
 
 from .graph_request import makeapirequestPost
+from .logger import log
 
 
 def create_batch_request(batch, batch_id, method, url, extra_url) -> tuple:
@@ -41,6 +42,7 @@ def handle_responses(
     """Handle the responses from the batch request.
 
     Args:
+        initial_request_data (list): List of initial requests
         request_data (list): List of responses from the batch request
         responses (list): List of responses from the batch request
         retry_pool (list): List of failed requests
@@ -53,10 +55,13 @@ def handle_responses(
         failed_batch_requests = []
         if resp["status"] == 200:
             responses.append(resp["body"])
+            retry_pool = [req for req in retry_pool if req["id"] != int(resp["id"])]
         elif resp["status"] in [429, 503]:
             if initial_request_data:
                 failed_batch_requests = [
-                    i for i in initial_request_data if i["id"] == int(resp["id"])
+                    i
+                    for i in initial_request_data
+                    if i["id"] == int(resp["id"]) and i not in retry_pool
                 ]
             retry_pool += failed_batch_requests
 
@@ -65,44 +70,101 @@ def handle_responses(
     return responses, retry_pool, wait_time
 
 
-def batch_request(data, url, extra_url, token, method="GET") -> list:
-    """_summary_
+def create_batch_list(data, batch_count) -> list:
+    """Create a list of batches from the data.
 
     Args:
-        data (list): List of IDs
+        data (list): List of objects
+        batch_count (int): Number of objects to include in each batch
+
+    Returns:
+        list: List of batches
+    """
+    return [data[i : i + batch_count] for i in range(0, len(data), batch_count)]
+
+
+def process_batch(
+    batch,
+    batch_id,
+    method,
+    url,
+    extra_url,
+    token,
+    initial_request_data,
+    responses,
+    retry_pool,
+) -> tuple:
+    """Process the batch request.
+
+    Args:
+        batch (list): List of objects
+        batch_id (str): ID for the batch request
+        method (str): HTTP method to use
         url (str): MS graph endpoint for the object
         extra_url (str): Used if anything extra is needed for the url such as /assignments or ?$filter
         token (str): OAuth token used for authentication
-        method (str, optional): HTTP method to use. Defaults to "GET".
+        initial_request_data (list): List of initial requests
+        responses (list): List of responses from the batch request
+        retry_pool (list): List of failed requests
 
     Returns:
-        list: List of responses from the batch request
+        tuple: Tuple containing the batch ID, responses, retry pool and wait time
     """
-    responses = []
-    batch_id = 1
-    batch_count = 20
-    retry_pool = []
-    wait_time = 0
-    batch_list = [data[i : i + batch_count] for i in range(0, len(data), batch_count)]
-    initial_request_data = []
+    query_data, batch_id = create_batch_request(batch, batch_id, method, url, extra_url)
+    json_data = json.dumps(query_data)
+    request = makeapirequestPost(
+        "https://graph.microsoft.com/beta/$batch", token, jdata=json_data
+    )
+    request_data = sorted(request["responses"], key=lambda item: item.get("id"))
+    initial_request_data += query_data["requests"]
+    responses, retry_pool, wait_time = handle_responses(
+        initial_request_data, request_data, responses, retry_pool
+    )
+    return batch_id, responses, retry_pool, wait_time
+
+
+def retry_failed_requests(
+    retry_pool,
+    wait_time,
+    max_retries,
+    max_wait_time,
+    token,
+    initial_request_data,
+    responses,
+    batch_count,
+) -> tuple:
+    """Retry failed requests.
 
-    for batch in batch_list:
-        query_data, batch_id = create_batch_request(
-            batch, batch_id, method, url, extra_url
-        )
-        json_data = json.dumps(query_data)
-        request = makeapirequestPost(
-            "https://graph.microsoft.com/beta/$batch", token, jdata=json_data
-        )
-        request_data = sorted(request["responses"], key=lambda item: item.get("id"))
-        initial_request_data += query_data["requests"]
-        responses, retry_pool, wait_time = handle_responses(
-            initial_request_data, request_data, responses, retry_pool
-        )
+    Args:
+        retry_pool (list): List of failed requests
+        wait_time (int): Time to wait before retrying
+        max_retries (int): Maximum number of retries
+        max_wait_time (int): Maximum time to wait before retrying
+        token (str): OAuth token used for authentication
+        initial_request_data (list): List of initial requests
+        responses (list): List of responses from the batch request
+        batch_count (int): Number of objects to include in each batch
 
-    if retry_pool:
+    Returns:
+        list: List of responses from the batch request
+    """
+    retry_count = 0
+    failed_retry_requests = []
+    while retry_count < max_retries and retry_pool:
+        log(
+            "retry_failed_requests",
+            f"Retrying failed requests, retry pool count: {str(len(retry_pool))}",
+        )
         if wait_time > 0:
+            log("retry_failed_requests", f"Sleeping for {str(wait_time)} seconds...")
             time.sleep(wait_time)
+            wait_time = min(wait_time * 2, max_wait_time)
+        else:
+            log(
+                "retry_failed_requests",
+                "No wait time in headers, sleeping for 20 seconds...",
+            )
+            time.sleep(20)
         batch_list = [
             retry_pool[i : i + batch_count]
             for i in range(0, len(retry_pool), batch_count)
@@ -114,9 +176,70 @@ def batch_request(data, url, extra_url, token, method="GET") -> list:
                 "https://graph.microsoft.com/beta/$batch", token, jdata=json_data
             )
             request_data = sorted(request["responses"], key=lambda item: item.get("id"))
-            responses, _, _ = handle_responses(
+            responses, retry_pool, _ = handle_responses(
                 initial_request_data, request_data, responses, retry_pool
             )
+            failed_retry_requests = [
+                r for r in request["responses"] if r["status"] != 200
+            ]
+        retry_count += 1
+        log("retry_failed_requests", f"Retry count: {str(retry_count)}")
+        if retry_pool and retry_count == max_retries:
+            break
+    log(
+        "retry_failed_requests",
+        f"Failed requests after {str(retry_count)} retries: {str(len(failed_retry_requests))}",
+    )
+    return responses
+
+
+def batch_request(data, url, extra_url, token, method="GET") -> list:
+    """Batch request to the Graph API.
+
+    Args:
+        data (list): List of objects
+        url (str): MS graph endpoint for the object
+        extra_url (str): Used if anything extra is needed for the url such as /assignments or ?$filter
+        token (str): OAuth token used for authentication
+        method (str): HTTP method to use
+
+    Returns:
+        list: List of responses from the batch request
+    """
+    responses = []
+    batch_id = 1
+    batch_count = 20
+    retry_pool = []
+    wait_time = 0
+    initial_request_data = []
+
+    batch_list = create_batch_list(data, batch_count)
+    for batch in batch_list:
+        batch_id, responses, retry_pool, wait_time = process_batch(
+            batch,
+            batch_id,
+            method,
+            url,
+            extra_url,
+            token,
+            initial_request_data,
+            responses,
+            retry_pool,
+        )
+
+    max_retries = 10
+    max_wait_time = 60
+    if retry_pool:
+        responses = retry_failed_requests(
+            retry_pool,
+            wait_time,
+            max_retries,
+            max_wait_time,
+            token,
+            initial_request_data,
+            responses,
+            batch_count,
+        )
 
     return responses
 

src/IntuneCD/intunecdlib/logger.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+This module is used to log messages if verbose is set to True.
+"""
+
+import os
+import time
+
+verbose = os.getenv("VERBOSE")
+
+
+def log(function, msg):
+    """Prints a message to the console if the VERBOSE environment variable is set to True.
+
+    Args:
+        function (str): The name of the function that is calling the log function.
+        msg (str): The message to print to the console.
+    """
+    if verbose:
+        msg = f"[{time.asctime()}] - [{function}] - {msg}"
+        print(msg)