generatedata.ipynb


from pyspark.sql import SparkSession
from pyspark.sql.functions import col
import random
import string

# Initialize SparkSession
spark = SparkSession.builder.appName("EmployeeDaatabasebyganeshkavhar").master("local[*]").getOrCreate()


# Function to generate random salaries
def random_salary():
    return random.randint(30000, 150000)

# Create a list of dictionaries with more than 100 employee records
employees = [{"id": i, "name": random_name(), "age": random.randint(20, 60), "department": random.choice(["HR", "Finance", "IT", "Marketing"]), "salary": random_salary()} for i in range(101)]

# Create a PySpark DataFrame from the list of dictionaries
df = spark.createDataFrame(employees)

# Show all records
# Approach 1: Using the show method with a large number
df.show(n=101, truncate=False)

# Approach 2: Convert to Pandas and display (for smaller datasets)
# pandas_df = df.toPandas()
# print(pandas_df)