Optimize Backend

sql-backend/Automation/API Fetch/TBA-Scraper.py

@@ -49,7 +49,7 @@ def AssembleMatchScrapes(match_keys):
 	for key in match_keys:
 		print("Assembling API scrape from key: " + str(key))
 		scrape = "/match/" + str(key) + "/simple"
-		print("Appending: " + str(scrape))
+		print("Appending [ " + str(scrape) + " ] to scrape list...")
 		scrapes.append(scrape)
 
 	time.sleep(1)
@@ -58,7 +58,7 @@ def AssembleMatchScrapes(match_keys):
 
 #Fetch a list of match keys from an event specified by the event key
 def FetchMatchKeys(event_key):
-	print("Fetching match keys...")
+	print("Fetching match keys from upstream...")
 	match_keys = getTBA("event/" + event_key + "/matches/keys")
 	print(match_keys)
 	print("Done!")
@@ -78,7 +78,8 @@ def FetchMatchData(match_scrapes):
 		match_data.append(getTBA(scrape))
 
 		current_stage = current_stage + 1
-		printProgressBar(current_stage, total_stages, prefix = 'Fetching: ' + str(scrape), suffix = 'Complete ', length = 50)
+		printProgressBar(current_stage, total_stages, prefix = 'Downloading: ' + str(scrape), suffix = 'Complete ', length = 50)
+
 
 	print(match_data)
 

sql-backend/Automation/SQL Builder/sqlBuilder.py

@@ -8,6 +8,7 @@
 
 #Imports from os
 import os
+import time
 from os import listdir
 
 #Imports from os.path
@@ -24,12 +25,12 @@
 ####################FUNCTIONS####################
 
 def ReadJSON():
-    # This function loops over every "*.scout" file and checks it's contents...
-    # If the file is in a valid JSON format, we will read it's elements and store them in a dictionary
-    # Once we finish storing the files elements we will add the dictionry to a list
-    # This function will return a list of dictionaries representing each files' JSON Data
-    # EXAMPLE RETURN: [{'firstFileOneInteger': 1, secondFileOneInteger: 2}, {file2Integer: 2}]
-    # NOTE: This function will only read the "*.scout" files in the current working directory
+	# This function loops over every "*.scout" file and checks it's contents...
+	# If the file is in a valid JSON format, we will read it's elements and store them in a dictionary
+	# Once we finish storing the files elements we will add the dictionry to a list
+	# This function will return a list of dictionaries representing each files' JSON Data
+	# EXAMPLE RETURN: [{'firstFileOneInteger': 1, secondFileOneInteger: 2}, {file2Integer: 2}]
+	# NOTE: This function will only read the "*.scout" files in the current working directory
 
 	files = os.listdir(CURRENT_WORKING_DIRECTORY) #Read all files from cwd and add to file list
 
@@ -41,28 +42,36 @@ def ReadJSON():
 				with open(file) as json_data: #Read json data of file into "json_data"
 					data = json.load(json_data) #Save loaded json data to a "data" dictionary
 					data_list.append(data) #Append the data dictionary to the list we defined above
+					print("Read data: [ " + str(data) + " ]")
 			except:
 				print("Unable to read \"*.scout\" file becasue it is not in a valid JSON format | ERROR") #Print an error message to the console'
 
+	print("Created Data List: " + str(data_list))
 	return data_list #Returns a list of dictionaries representing each files' JSON Data
 
 
 def RemoveBadCharacters(string):
-    # This function is responsible for removing bad SQL characters like ' and "
-
-    output = "" # Define an output string
-    for char in string: # Loop through each character in our string
-        if(char != "'" and char != "\""): # If the string does not contain an invalid character
-            output = output + char # Add the current character to our output
-
-    return output # Return the output string
-
+	print("Removing reserved SQL Characters")
+
+	# This function is responsible for removing bad SQL characters like ' and "
+
+	output = "" # Define an output string
+	for char in string: # Loop through each character in our string
+		if(char != "'" and char != "\""): # If the string does not contain an invalid character
+			output = output + char # Add the current character to our output
+			print("Appending Character [ " + str(char) + " ] to statement...")
+		else:
+			print("Invalid Character: " + str(char) + "! Will not append!")
+
+	print("Operation Complete! Resulting Data Entry: [ " + str(output) + " ] ")
+	return output # Return the output string
+
 def BuildInsertQueries(data):
 	# This function builds SQL queries from each dictionary KeyValuePair in a list of dictionaries
 	# This function will loop over every JSON dictionary in the list of dictionaries and then loop over Each KeyValuePair in the dictionary
 	# We will then generate an insert querry by taking using the key as the column and the value as the data point
 	# EXAMPLE QUERY: INSERT INTO MatchMaster1 (ScoutedBy)VALUES(John Doe)
-    
+	
 	query_list = [] #Define an empty list to store SQL querries in
 	database = "[Scouting 2022].[dbo].[" + str(TABLE_NAME) + "]" #Store the name of the table to be referenced again later when creating the query
 
@@ -91,11 +100,13 @@ def BuildInsertQueries(data):
 
 		query = query[:-1] #Exclude last comma generated in string
 		query = query + ")" #Add final closing parenthiss to query
-
+
+		print("Generated SQL Query [ " + str(query) + " ] Appending...")
 		query_list.append(query) #Append the query we created to the query_list
-
+
+	print("Resulting Query List: " + str(query_list))
 	return query_list #Return the list of queries that we have created
-    
+	
 def WriteQueryFile(query_list):
 
 	# This function loops through all the queries in the query list and appends them to a string
@@ -107,18 +118,18 @@ def WriteQueryFile(query_list):
 		fileContents = ("--This SQL query file was built with the SQL Builder python script\n\n")
 
 		for query in query_list:
-			fileContents = fileContents + (str(query)) + "-- Generated Query for *.scout file\n"
+			print("Writing Query [ " + str(query) + " ] to TSQL query batch...\n\n")
+			fileContents = fileContents + (str(query)) + "-- Generated Query for *.scout file\n\n"
 
+		print("Resulting File Contents: " + str(fileContents) + "\n\n")
 		builderOutputFile.write(fileContents)
 
-def RunSQLQuery():
-	cwd = os.getcwd() #Get current working directory (cwd) of this file
-	subprocess.call(CURRENT_WORKING_DIRECTORY + BUILDER_OUTPUT_FILE_NAME)
-
 ####################MAIN####################
-    
+time.sleep(1)
 data = ReadJSON()
+time.sleep(1)
 queries = BuildInsertQueries(data)
+time.sleep(1)
 WriteQueryFile(queries)
 
-#	RunSQLQuery()
+print("Complete! Press any key to close this window!")

sql-backend/Automation/Scraper - Parser - Mover - Querrier.cmd

@@ -16,9 +16,9 @@ cd %~dp0
 cls
 
 
-echo Running API Fetch/runTBA.cmd...
+echo Executing "API Fetch/TBA-Scraper.py"
 cd "API Fetch"
-start /W /max runTBA.cmd
+python TBA-Scraper.py
 cd ..
 
 echo Executing "VB Scripts/TeamMover.vbs"
@@ -31,7 +31,9 @@ cd "VB Scripts"
 start /W /max GSON_Converter.vbs
 cd ..
 
-echo Executing "SQL Builder/run-py.cmd"
+echo Executing "SQL Builder/sqlBuilder.py"
 cd "SQL Builder"
-start /W /max run-py.cmd
-cd ..
+python sqlBuilder.py
+cd ..
+
+pause > nul