Obfuscation.py

"""
PerBlur is extension of previous work proposed by Windenberg et al., (BlurMe: Inferring and Obfuscating
User Gender Based on Ratings ) and Strucks et al., (BlurM(or)e: Revisiting Gender Obfuscation
in the User-Item Matrix)

This code is extending previous github repository done by Christopher Strucks (Github Link: https://github.com/STrucks/BlurMore)

In PerBlur you need to :
    + Generate json file: "Confidence score" from imputation/knn/few_observed_entries
    + You will read the json file
"""

import json

from sklearn.model_selection import GridSearchCV

import MovieLensData as MD
import numpy as np
import Utils
import Classifiers
import matplotlib.pyplot as plt
import scipy.stats as ss
import pandas as pd
import json

class NpEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.integer):
            return int(obj)
        elif isinstance(obj, np.floating):
            return float(obj)
        elif isinstance(obj, np.ndarray):
            return obj.tolist()
        else:
            return super(NpEncoder, self).default(obj)

## ML1M For BlurMe
# L_m = [589.0,1233.0,2706.0,1090.0,2959.0,1250.0,2427.0,2490.0,1208.0,1266.0,3654.0,1748.0,1262.0,1396.0,1374.0,2700.0,1265.0,1089.0,1222.0,231.0,2770.0,1676.0,2890.0,1228.0,1136.0,3360.0,3298.0,1663.0,3811.0,2011.0,1261.0,233.0,3361.0,2366.0,1127.0,1276.0,3555.0,1214.0,3929.0,299.0,1304.0,3468.0,1095.0,150.0,1213.0,750.0,3082.0,6.0,111.0,3745.0,349.0,541.0,2791.0,785.0,1060.0,1294.0,1302.0,2012.0,1256.0,1292.0,2948.0,3683.0,3030.0,3836.0,913.0,2150.0,32.0,2826.0,2721.0,590.0,3623.0,2997.0,3868.0,2944.0,3147.0,1610.0,3508.0,2046.0,21.0,1249.0,10.0,1283.0,3760.0,2712.0,3617.0,3421.0,3552.0,3256.0,1079.0,3053.0,1517.0,3196.0,2662.0,1953.0,2670.0,3578.0,2371.0,3334.0,2028.0,2502.0,2951.0,2278.0,364.0,3462.0,924.0,2401.0,3163.0,2311.0,852.0,2916.0,1378.0,3384.0,1046.0,524.0,70.0,370.0,3035.0,3513.0,2917.0,3697.0,24.0,1957.0,3494.0,1912.0,3752.0,2013.0,3452.0,3928.0,2987.0,431.0,2759.0,1387.0,1882.0,1274.0,3327.0,3638.0,1288.0,2867.0,2728.0,2921.0,2433.0,161.0,3386.0,517.0,741.0,1287.0,1231.0,3062.0,2288.0,3753.0,529.0,3793.0,3052.0,2447.0,1320.0,3819.0,1303.0,922.0,3022.0,260.0,2640.0,858.0,493.0,3006.0,480.0,2410.0,333.0,1178.0,3814.0,2702.0,1203.0,2922.0,2761.0,1625.0,3366.0,223.0,3213.0,996.0,2188.0,2628.0,3358.0,2648.0,3788.0,953.0,999.0,3754.0,3910.0,3016.0,3863.0,303.0,3263.0,1080.0,786.0,3764.0,2105.0,3543.0,3018.0,2607.0,3681.0,592.0,145.0,2303.0,1682.0,553.0,1019.0,1272.0,3646.0,1544.0,235.0,908.0,3615.0,2792.0,3354.0,1204.0,2167.0,2455.0,1587.0,3629.0,1227.0,2901.0,2687.0,1883.0,104.0,1210.0,1201.0,3169.0,3098.0,3688.0,2409.0,3198.0,610.0,1923.0,1982.0,2501.0,165.0,2403.0,784.0,2871.0,2889.0,628.0,2300.0,417.0,3671.0,3100.0,1221.0,3914.0,1218.0,3608.0,3152.0,3429.0,3265.0,1794.0,952.0,1391.0,2518.0,410.0,3535.0,2333.0,1713.0,1263.0,3947.0,2605.0,707.0,2973.0,2795.0,1965.0,373.0,2970.0,3916.0,556.0,3703.0,95.0,466.0,2060.0,3066.0,3177.0,368.0,2088.0,1476.0,163.0,3422.0,58.0,1244.0,1689.0,2002.0,1929.0,599.0,1711.0,2259.0,1209.0,3524.0,1977.0,1371.0,3506.0,3104.0,3444.0,1693.0,965.0,1732.0,2600.0,3424.0,3755.0,2450.0,3826.0,2301.0,3801.0,3927.0,1298.0,198.0,2118.0,112.0,2478.0,471.0,1673.0,1246.0,2734.0,2529.0,2806.0,2094.0,1948.0,3710.0,2093.0,45.0,648.0,3504.0,2968.0,1722.0,1963.0,2840.0,1747.0,2476.0,1348.0,2411.0,3871.0,3175.0,2360.0,1092.0,3190.0,3729.0,1405.0,1007.0,2034.0,492.0,367.0,3248.0,3806.0,537.0,3619.0,1702.0,3210.0,1734.0,2644.0,2252.0,1597.0,1200.0,1401.0,1416.0,107.0,383.0,1379.0,2764.0,2116.0,1036.0,60.0,2682.0,2115.0,1317.0,1876.0,3394.0,1254.0,2243.0,2606.0,3925.0,1503.0,3869.0,1612.0,390.0,328.0,3903.0,3375.0,2422.0,3087.0,1627.0,3770.0,273.0,3430.0,3678.0,3576.0,2377.0,3113.0,196.0,3036.0,3525.0,1584.0,799.0,2236.0,3267.0,954.0,2412.0,1690.0,1879.0,30.0,1205.0,1252.0,2470.0,2686.0,3397.0,2367.0,2015.0,3804.0,3002.0,1377.0,3740.0,1594.0,1978.0,2456.0,3821.0,2038.0,891.0,1342.0,2669.0,3628.0,1966.0,2808.0,3324.0,366.0,3634.0,3794.0,3509.0,1063.0,2467.0,3420.0,324.0,3773.0,1927.0,1241.0,2231.0,3742.0,1960.0,1390.0,1542.0,2672.0,83.0,1376.0,3174.0,2395.0,1248.0,3362.0,2029.0,225.0,2362.0,2491.0,110.0,3527.0,1267.0,3203.0,511.0,1025.0,2769.0,3792.0,3785.0,1973.0,1952.0,326.0,2541.0,3593.0,3698.0,2058.0,2459.0,1375.0,3273.0,1278.0,154.0,1179.0,2009.0,1976.0,300.0,735.0,504.0,2423.0,1365.0,2676.0,3300.0,3701.0,2793.0,2915.0,963.0,3505.0,2178.0,1920.0,3934.0,813.0,3357.0,2580.0,2430.0,1975.0,1695.0,2542.0,1701.0,2195.0,2560.0,1215.0,637.0,3252.0,440.0,100.0,540.0,3091.0,2654.0,1885.0,2372.0,2384.0,1979.0,3507.0,1414.0,1251.0,1187.0,3316.0,1609.0,2841.0,2149.0,1986.0,2624.0,1305.0,2287.0,1858.0,2004.0,1260.0,1687.0,1785.0,1257.0,516.0,3725.0,1358.0,2253.0,3918.0,157.0,2374.0,2976.0,1100.0,1866.0,3802.0,2140.0,1646.0,737.0,2092.0,2165.0,1604.0,2772.0,2249.0,3500.0,193.0,3037.0,665.0,1196.0,3728.0,194.0,555.0,1431.0,3851.0,3564.0,2322.0,1370.0,3379.0,3099.0,3188.0,321.0,2863.0,2492.0,13.0,3840.0,2378.0,1255.0,3271.0,3946.0,1550.0,1017.0,3086.0,3189.0,1445.0,3074.0,3908.0,3000.0,1837.0,2296.0,340.0,2306.0,1238.0,3551.0,1389.0,3689.0,1884.0,800.0,3693.0,2526.0,2327.0,3180.0,387.0,1193.0,3254.0,3614.0,2829.0,459.0,448.0,3033.0,1779.0,458.0,3477.0,1388.0,1289.0,748.0,3289.0,1024.0,1411.0,2458.0,3569.0,2836.0,3948.0,1057.0,2877.0,2633.0,3078.0,2289.0,514.0,3831.0,3733.0,3311.0,535.0,361.0,290.0,2027.0,1408.0,1198.0,948.0,1356.0,2522.0,1859.0,2321.0,2356.0,2660.0,3017.0,1245.0,1985.0,1395.0,1340.0,1103.0,2781.0,3812.0,2861.0,544.0,1974.0,474.0,3276.0,2497.0,2323.0,3304.0,1980.0,2851.0,3682.0,1981.0,1633.0,2620.0,3068.0,1129.0,304.0,2457.0,2530.0,1931.0,125.0,548.0,2148.0,3487.0,2651.0,1735.0,2896.0,3632.0,89.0,680.0,2113.0,3519.0,1008.0,2387.0,2912.0,325.0,3159.0,171.0,3355.0,892.0,2593.0,990.0,3669.0,1051.0,1301.0,1983.0,711.0,2828.0,2924.0,523.0,3368.0,2521.0,3148.0,296.0,2488.0,1867.0,1382.0,3917.0,2626.0,2883.0,522.0,3846.0,2882.0,1639.0,3015.0,3639.0,2212.0,1665.0,3834.0,1297.0,829.0,15.0,2083.0,534.0,2942.0,2898.0,1247.0,3008.0,2663.0,861.0,1180.0,2817.0,155.0,3266.0,3039.0,2107.0,419.0,761.0,3469.0,3544.0,2210.0,1970.0,3338.0,3307.0,1161.0,3048.0,1054.0,432.0,2796.0,2425.0,3879.0,2117.0,3747.0,3930.0,685.0,2525.0,742.0,1128.0,204.0,3709.0,3417.0,2656.0,3937.0,1844.0,1237.0,2885.0,3073.0,888.0,2181.0,1091.0,3214.0,3880.0,1273.0,315.0,3809.0,436.0,3115.0,3387.0,2111.0,3694.0,468.0,2775.0,1752.0,3301.0,2810.0,2347.0,1696.0,2837.0,1341.0,2023.0,1591.0,565.0,3769.0,2961.0,518.0,788.0,2134.0,3120.0,3771.0,1581.0,246.0,3767.0,1130.0,44.0,63.0,527.0,893.0,3626.0,2843.0,3590.0,456.0,3817.0,487.0,1726.0,2645.0,810.0,860.0,1058.0,3529.0,3094.0,2907.0,2334.0,2344.0,2641.0,3230.0,2271.0,1016.0,151.0,377.0,1494.0,2402.0,450.0,3538.0,3717.0,2694.0,2745.0,2389.0,3780.0,3141.0,3865.0,281.0,2472.0,3441.0,3272.0,3734.0,2255.0,850.0,1750.0,2507.0,2272.0,2991.0,3021.0,3498.0,2983.0,3766.0,1810.0,1437.0,2024.0,842.0,1184.0,2725.0,1488.0,2064.0,2731.0,1999.0,2139.0,2965.0,1078.0,409.0,3692.0,3758.0,1924.0,2019.0,1596.0,137.0,760.0,2971.0,569.0,897.0,305.0,1083.0,2238.0,2701.0,1753.0,3364.0,3643.0,3351.0,2404.0,2597.0,1459.0,2835.0,3680.0,1945.0,2567.0,673.0,3568.0,3516.0,3830.0,2726.0,2515.0,611.0,947.0,121.0,568.0,1176.0,1998.0,1640.0,1626.0,3032.0,1601.0,835.0,1739.0,2755.0,388.0,484.0,1279.0,1326.0,172.0,2787.0,2977.0,3596.0,2939.0,1855.0,3306.0,3372.0,1865.0,1725.0,3241.0,1432.0,3089.0,2179.0,870.0,2671.0,944.0,2325.0,968.0,2535.0,2041.0,2802.0,2723.0,882.0,3695.0,2985.0,3658.0,2471.0,3365.0,1516.0,1113.0,667.0,378.0,2981.0,3415.0,3401.0,764.0,2897.0,297.0,3810.0,210.0,2773.0,1787.0,3901.0,268.0,3339.0,3402.0,2647.0,3221.0,1450.0,416.0,719.0,3385.0,3183.0,528.0,2169.0,1834.0,3490.0,1733.0,3069.0,178.0,1135.0,2929.0,2460.0,2751.0,2780.0,998.0,2475.0,20.0,292.0,3919.0,255.0,505.0,811.0,88.0,3920.0,3347.0,547.0,1527.0,3622.0,479.0,2121.0,2852.0,3952.0,424.0,1219.0,1869.0,413.0,393.0,567.0,2866.0,2264.0,2055.0,214.0,3898.0,3803.0,3737.0,921.0,3034.0,2294.0,3935.0,1366.0,1154.0,3412.0,3395.0,322.0,354.0,3659.0,2675.0,1997.0,849.0,2017.0,2615.0,1322.0,118.0,2025.0,2986.0,3873.0,1889.0,442.0,1793.0,496.0,3476.0,259.0,1399.0,3807.0,3233.0,831.0,1992.0,2061.0,2536.0,3822.0,3416.0,2652.0,2086.0,2283.0,3520.0,694.0,1890.0,2822.0,1458.0,2068.0,681.0,180.0,3690.0,2673.0,2482.0,3933.0,2040.0,423.0,885.0,1330.0,116.0,3572.0,1329.0,762.0,3146.0,159.0,2076.0,1677.0,3239.0,2462.0,3700.0,2842.0,1783.0,2634.0,3759.0,2979.0,1860.0,2328.0,3326.0,3432.0,3319.0,1655.0,1760.0,2437.0,632.0,3893.0,275.0,2594.0,16.0,2164.0,1351.0,1153.0,2244.0,1942.0,2171.0,3774.0,2815.0,3818.0,3392.0,2073.0,101.0,557.0,3178.0,3491.0,1120.0,2753.0,1281.0,2665.0,1589.0,1468.0,274.0,936.0,3585.0,1901.0,1635.0,1535.0,394.0,330.0,1880.0,2566.0,2326.0,2119.0,2591.0,2596.0,3765.0,1406.0,3168.0,443.0,2215.0,202.0,1085.0,2627.0,3143.0,3603.0,747.0,3106.0,2520.0,3095.0,1670.0,3031.0,2196.0,2905.0,1551.0,1439.0,3503.0,3448.0,793.0,2914.0,1971.0,437.0,1910.0,1230.0,3129.0,3636.0,2993.0,2189.0,3847.0,3942.0,3820.0,3799.0,1355.0,2809.0,1780.0,3125.0,2839.0,3784.0,287.0,3149.0,3845.0,2003.0,2609.0,3789.0,2275.0,731.0,1623.0,3262.0,1746.0,566.0,1532.0,2408.0,355.0,841.0,1044.0,3670.0,3537.0,181.0,3648.0,3726.0,1170.0,3691.0,2350.0,464.0,12.0,3816.0,2376.0,3602.0,3309.0,623.0,2407.0,2137.0,1946.0,1574.0,3939.0,1420.0,2390.0,2707.0,1519.0,1543.0,1504.0,1190.0,1851.0,1771.0,2473.0,2902.0,2537.0,3275.0,2441.0,2748.0,3727.0,2544.0,1918.0,122.0,2847.0,1533.0,2098.0,1474.0,3425.0,3020.0,486.0,3201.0,3768.0,780.0,3909.0,1632.0,1791.0,3473.0,3139.0,710.0,725.0,1583.0,1853.0,3367.0,2512.0,76.0,1160.0,3673.0,1826.0,2943.0,2082.0,1886.0,749.0,2571.0,2830.0,1472.0,3157.0,2170.0,293.0,3938.0,3154.0,1895.0,2256.0,2187.0,2112.0,2237.0,3407.0,1846.0,513.0,716.0,3731.0,3875.0,3409.0,840.0,2286.0,1692.0,2913.0,925.0,1529.0,1703.0,2351.0,177.0,1606.0,1772.0,3587.0,1680.0,663.0,503.0,1325.0,2528.0,828.0,263.0,3884.0,2610.0,2483.0,2201.0,3012.0,564.0,2295.0,743.0,704.0,3633.0,2153.0,3049.0,3322.0,434.0,2418.0,607.0,1915.0,2737.0,2618.0,626.0,2192.0,3718.0,1404.0,3610.0,1490.0,66.0,2391.0,1497.0,209.0,336.0,941.0,1684.0,1741.0,3.0,3651.0,1253.0,2994.0,3532.0,3775.0,720.0,2383.0,1955.0,967.0,3652.0,970.0,433.0,2960.0,1715.0,1493.0,674.0,3713.0,937.0,463.0,1531.0,2349.0,851.0,572.0,126.0,3310.0,744.0,1415.0,2332.0,2175.0,1599.0,563.0,585.0,3514.0,703.0,3140.0,2353.0,2480.0,3744.0,158.0,2740.0,9.0,3349.0,3676.0,3398.0,3075.0,3025.0,1743.0,1598.0,1385.0,162.0,485.0,1454.0,2200.0,881.0,59.0,2335.0,765.0,3777.0,1284.0,2894.0,2089.0,133.0,3123.0,1148.0,2659.0,1003.0,2666.0,1147.0,3047.0,536.0,2523.0,3427.0,1053.0,989.0,2183.0,2499.0,2767.0,3661.0,2818.0,430.0,2400.0,1037.0,2804.0,1731.0,1142.0,2419.0,3118.0,2234.0,1482.0,1679.0,2209.0,2868.0,3313.0,1068.0,987.0,702.0,1006.0,854.0,1704.0,2393.0,1412.0,3523.0,3228.0,755.0,3294.0,2250.0,2211.0,2045.0,2945.0,2246.0,1369.0,1144.0,1567.0,2268.0,3076.0,426.0,2869.0,3890.0,2589.0,2554.0,384.0,3640.0,630.0,679.0,2524.0,460.0,2302.0,1984.0,3536.0,2988.0,1617.0,1138.0,1038.0,445.0,2911.0,71.0,1457.0,561.0,365.0,2207.0,1539.0,1335.0,3374.0,1603.0,617.0,734.0,3135.0,2001.0,2689.0,3121.0,53.0,81.0,3511.0,2824.0,796.0,1891.0,2936.0,3431.0,3142.0,2668.0,3110.0,3331.0,1928.0,815.0,1652.0,103.0,3167.0,2754.0,3842.0,2935.0,3280.0,2510.0,1436.0,78.0,2819.0,1728.0,1871.0,2705.0,1842.0,3470.0,526.0,3940.0,1461.0,3776.0,1650.0,241.0,863.0,2744.0,889.0,510.0,533.0,3531.0,759.0,175.0,1149.0,439.0,2834.0,980.0,1194.0,3832.0,729.0,3762.0,2545.0,3380.0,2232.0,3295.0,478.0,2095.0,56.0,886.0,3860.0,3749.0,2667.0,3573.0,2317.0,2538.0,2557.0,2887.0,992.0,1040.0,1830.0,766.0,1718.0,3127.0,2646.0,1861.0,2102.0,818.0,3116.0,3042.0,1425.0,228.0,188.0,2341.0,3371.0,1426.0,3465.0,643.0,3320.0,909.0,3563.0,1903.0,3867.0,570.0,878.0,1282.0,184.0,3732.0,728.0,331.0,2185.0,3092.0,1475.0,3187.0,313.0,3521.0,2813.0,3410.0,2451.0,14.0,3601.0,3208.0,1972.0,3400.0,1744.0,1350.0,3783.0,3282.0,3756.0,1334.0,1361.0,1291.0,2638.0,874.0,664.0,1164.0,2318.0,551.0,2416.0,2904.0,1925.0,2760.0,3013.0,3575.0,2858.0,2221.0,2602.0,2223.0,2999.0,2265.0,847.0,2878.0,2394.0,2623.0,1098.0,1121.0,1849.0,1538.0,598.0,3346.0,3588.0,3342.0,809.0,3800.0,3627.0,3905.0,3850.0,2509.0,2032.0,1528.0,573.0,3182.0,244.0,3096.0,2071.0,19.0,1168.0,1664.0,3235.0,1839.0,2658.0,310.0,2016.0,227.0,867.0,2166.0,1887.0,3056.0,1647.0,1062.0,1817.0,3344.0,33.0,696.0,3216.0,2611.0,1864.0,2204.0,1502.0,3433.0,545.0,3735.0,1433.0,3205.0,3434.0,1234.0,2486.0,3442.0,682.0,2290.0,3736.0,2415.0,1004.0,1071.0,454.0,2632.0,2582.0,3381.0,1534.0,3779.0,3751.0,1681.0,1825.0,3889.0,3389.0,782.0,3837.0,3084.0,3900.0,2397.0,1553.0,2691.0,757.0,666.0,3184.0,2239.0,3318.0,2708.0,3647.0,1907.0,3292.0,3482.0,190.0,3172.0,96.0,791.0,961.0,3876.0,75.0,633.0,1991.0,1455.0,3533.0,130.0,2534.0,401.0,2794.0,758.0,2129.0,684.0,690.0,2735.0,3778.0,3377.0,2452.0,3207.0,1471.0,288.0,2176.0,1675.0,2124.0,1770.0,2310.0,3333.0,2674.0,23.0,3891.0,3548.0,1039.0,2821.0,1466.0,3404.0,672.0,977.0,3054.0,488.0,3212.0,3314.0,789.0,3743.0,958.0,2825.0,1969.0,981.0,385.0,1878.0,642.0,1102.0,1510.0,792.0,2776.0,1124.0,1125.0,2811.0,2033.0,2736.0,1795.0,3305.0,1590.0,823.0,706.0,3443.0,2191.0,2138.0,884.0,820.0,3687.0,2730.0,655.0,578.0,2865.0,3242.0,2533.0,1872.0,2292.0,3464.0,3772.0,3522.0,2716.0,2343.0,120.0,1816.0,3337.0,2932.0,1649.0,3463.0,2649.0,1189.0,428.0,312.0,614.0,827.0,2955.0,580.0,718.0,67.0,3283.0,1119.0,3485.0,1558.0,508.0,3653.0,601.0,1764.0,1908.0,2584.0,3026.0,582.0,1315.0,657.0,660.0,3437.0,3158.0,2281.0,1134.0,2135.0,2163.0,3040.0,3458.0,3243.0,3887.0,2242.0,467.0,138.0,530.0,584.0,701.0,3229.0,2758.0,2563.0,2749.0,402.0,574.0,1152.0,895.0,3782.0,1548.0,3070.0,1799.0,3359.0,1339.0,311.0,3151.0,123.0,1055.0,1224.0,1989.0,2996.0,2539.0,3085.0,1123.0,1470.0,1619.0,2578.0,658.0,2463.0,344.0,453.0,1905.0,1101.0,77.0,3236.0,515.0,576.0,2152.0,3297.0,3440.0,1714.0,2428.0,1520.0,183.0,2713.0,3574.0,1486.0,826.0,763.0,1070.0,2214.0,579.0,644.0,651.0,859.0,1118.0,1145.0,1630.0,1852.0,2235.0,2277.0,2484.0,2742.0,3220.0,3321.0,962.0,3403.0,2131.0,3597.0,1873.0,3941.0,3317.0,2308.0,1364.0,2895.0,3828.0,991.0,226.0,2879.0,200.0,1392.0,730.0,771.0,3888.0,3488.0,1165.0,2592.0,1651.0,3644.0,1877.0,2474.0,3666.0,3057.0,3277.0,2556.0,1820.0,2850.0,3001.0,790.0,3130.0,2420.0,717.0,774.0,1511.0,2812.0,142.0,1115.0,1386.0,3312.0,3621.0,3373.0,3237.0,93.0,1133.0,2386.0,2583.0,649.0,258.0,3065.0,1440.0,2636.0,1773.0,1863.0,1360.0,3904.0,2254.0,3797.0,3291.0,2519.0,3722.0,993.0,1293.0,2218.0,1613.0,132.0,1724.0,868.0,3336.0,583.0,865.0,638.0,316.0,1430.0,1993.0,732.0,843.0,975.0,2213.0,2217.0,3202.0,3376.0,2880.0,3378.0,2771.0,814.0,1217.0,521.0,3460.0,853.0,3945.0,3899.0,1419.0,135.0,2579.0,472.0,213.0,167.0,2527.0,2240.0,374.0,3486.0,250.0,2493.0,3419.0,451.0,2906.0,964.0,2552.0,2313.0,1809.0,1526.0,3253.0,1258.0,2984.0,2364.0,1310.0,2893.0,1343.0,2661.0,1212.0,3611.0,1935.0,2643.0,2947.0,2982.0,714.0,3637.0,1501.0,836.0,229.0,2941.0,3439.0,147.0,80.0,3258.0,896.0,745.0,3674.0,2159.0,923.0,1270.0,2461.0,2729.0,2637.0,1464.0,2798.0,353.0,1332.0,1707.0,1240.0,502.0,2859.0,3600.0,2108.0,973.0,1359.0]
# L_f = [920.0,3844.0,2369.0,1088.0,3534.0,1207.0,17.0,1041.0,3512.0,3418.0,1188.0,902.0,2336.0,3911.0,1441.0,141.0,2690.0,265.0,928.0,39.0,2762.0,906.0,838.0,2657.0,2125.0,3565.0,1967.0,2291.0,914.0,932.0,1620.0,2160.0,247.0,222.0,261.0,2881.0,2145.0,3072.0,1028.0,1956.0,2080.0,1286.0,3798.0,1959.0,28.0,2248.0,3247.0,3594.0,3155.0,1345.0,531.0,1277.0,593.0,3044.0,3083.0,3005.0,1296.0,1380.0,2020.0,105.0,1678.0,1608.0,2572.0,3791.0,1104.0,2144.0,318.0,1186.0,1073.0,595.0,2724.0,1641.0,351.0,2908.0,357.0,3079.0,1688.0,3556.0,3186.0,2406.0,224.0,1962.0,1480.0,3251.0,11.0,345.0,3526.0,1784.0,951.0,3668.0,2485.0,1958.0,2739.0,916.0,950.0,2443.0,3684.0,236.0,904.0,898.0,587.0,552.0,339.0,2143.0,3481.0,3097.0,3067.0,1449.0,47.0,36.0,616.0,3281.0,1259.0,661.0,2348.0,562.0,3606.0,2496.0,2085.0,1271.0,372.0,2857.0,3325.0,1394.0,1081.0,1032.0,918.0,1409.0,314.0,899.0,733.0,2245.0,381.0,2316.0,232.0,2405.0,2677.0,1066.0,2396.0,2282.0,1059.0,2622.0,1941.0,959.0,3479.0,3124.0,1197.0,1777.0,915.0,955.0,1648.0,3705.0,3061.0,22.0,34.0,926.0,1285.0,1.0,2875.0,1150.0,3545.0,2664.0,2155.0,1097.0,262.0,3915.0,971.0,2581.0,2186.0,3702.0,3105.0,2280.0,3604.0,3515.0,1513.0,2331.0,1500.0,2803.0,945.0,2639.0,3051.0,837.0,3408.0,457.0,1801.0,2506.0,4.0,2469.0,270.0,46.0,1235.0,2355.0,2346.0,1357.0,461.0,3255.0,3176.0,3350.0,2975.0,2014.0,3936.0,2072.0,1353.0,2006.0,1397.0,2612.0,1099.0,1367.0,3270.0,938.0,2357.0,94.0,412.0,1518.0,3591.0,538.0,2000.0,2846.0,708.0,329.0,2995.0,653.0,1280.0,5.0,337.0,588.0,2546.0,1022.0,2692.0,2468.0,1569.0,905.0,1031.0,900.0,2565.0,1541.0,2926.0,2312.0,3730.0,1900.0,2718.0,1021.0,597.0,3185.0,2746.0,327.0,2805.0,3101.0,2920.0,289.0,3269.0,1674.0,477.0,3686.0,2077.0,2801.0,581.0,2133.0,3257.0,3724.0,3296.0,3554.0,3478.0,1479.0,3720.0,491.0,1014.0,1236.0,3134.0,695.0,1645.0,2763.0,1013.0,1096.0,1856.0,2827.0,248.0,1875.0,3211.0,3672.0,215.0,3224.0,3396.0,469.0,1897.0,3528.0,2870.0,917.0,930.0,1654.0,1009.0,3328.0,3786.0,1307.0,907.0,3870.0,1422.0,2206.0,2114.0,2324.0,2575.0,919.0,3467.0,1047.0,1806.0,350.0,230.0,2505.0,48.0,182.0,1537.0,144.0,170.0,2141.0,1916.0,3081.0,1191.0,1086.0,2598.0,805.0,546.0,3179.0,1407.0,153.0,2635.0,2057.0,2037.0,1327.0,3436.0,3145.0,446.0,2193.0,3341.0,942.0,1337.0,1913.0,195.0,2132.0,1804.0,3562.0,3706.0,1172.0,1042.0,2946.0,317.0,2514.0,3112.0,1093.0,1035.0,3499.0,1616.0,3011.0,2151.0,1111.0,1545.0,613.0,1043.0,2774.0,2154.0,2621.0,52.0,3060.0,2617.0,3723.0,206.0,3133.0,1821.0,1964.0,2504.0,211.0,2454.0,532.0,218.0,3156.0,43.0,1586.0,1126.0,2096.0,455.0,927.0,2007.0,778.0,2097.0,3117.0,539.0,691.0,943.0,1947.0,3567.0,1223.0,2194.0,2081.0,1268.0,1300.0,2747.0,1892.0,830.0,1914.0,1573.0,3302.0,671.0,3471.0,3825.0,3111.0,1064.0,1299.0,252.0,3004.0,2091.0,931.0,2337.0,605.0,1183.0,61.0,1020.0,3763.0,1727.0,74.0,3599.0,3708.0,465.0,29.0,3741.0,3457.0,2399.0,781.0,69.0,3635.0,3808.0,650.0,3249.0,2732.0,1621.0,1686.0,3435.0,3857.0,3299.0,3426.0,1225.0,2247.0,176.0,343.0,2972.0,2853.0,1033.0,272.0,2788.0,1393.0,2918.0,2432.0,3285.0,203.0,1465.0,801.0,1917.0,2431.0,342.0,3714.0,2967.0,3553.0,79.0,3951.0,1683.0,3071.0,3102.0,302.0,497.0,3655.0,2261.0,237.0,3260.0,3877.0,2174.0,2266.0,3716.0,3699.0,2782.0,1769.0,266.0,1173.0,2693.0,3093.0,1029.0,253.0,1658.0,277.0,279.0,848.0,839.0,2365.0,2738.0,1264.0,271.0,1643.0,1269.0,2043.0,3855.0,1030.0,1346.0,2104.0,2052.0,2161.0,2142.0,2719.0,2574.0,2053.0,1410.0,3912.0,1381.0,3660.0,2446.0,2613.0,2314.0,1132.0,978.0,1498.0,348.0,2168.0,3466.0,669.0,3649.0,2448.0,2899.0,1611.0,2940.0,362.0,8.0,1463.0,26.0,3557.0,1994.0,1758.0,414.0,1027.0,2681.0,3088.0,3483.0,140.0,3835.0,3391.0,1936.0,2205.0,2123.0,3861.0,332.0,3450.0,2585.0,2050.0,3618.0,425.0,709.0,1605.0,276.0,3827.0,846.0,2267.0,2359.0,2952.0,2786.0,3923.0,1939.0,1290.0,3240.0,3388.0,1547.0,338.0,3712.0,722.0,3063.0,242.0,715.0,3679.0,3571.0,3286.0,668.0,1069.0,2276.0,1438.0,2511.0,2688.0,1657.0,2900.0,3161.0,168.0,3539.0,199.0,2888.0,1624.0,3675.0,2436.0,647.0,724.0,82.0,542.0,1362.0,2424.0,117.0,2208.0,2109.0,3246.0,3019.0,3605.0,1904.0,360.0,267.0,2966.0,3620.0,700.0,3122.0,482.0,2741.0,334.0,2100.0,500.0,2173.0,1615.0,358.0,280.0,3932.0,369.0,3547.0,3739.0,3132.0,3849.0,1592.0,1788.0,875.0,2106.0,3719.0,3107.0,1423.0,3546.0,3839.0,1417.0,3566.0,3795.0,670.0,933.0,1177.0,520.0,2146.0,152.0,208.0,1175.0,3449.0,3274.0,27.0,3872.0,2733.0,207.0,2969.0,2927.0,957.0,2442.0,113.0,2078.0,2923.0,2084.0,1585.0,877.0,1807.0,1848.0,3882.0,3790.0,3926.0,2820.0,1629.0,3922.0,1571.0,3046.0,832.0,3896.0,2101.0,1600.0,3510.0,1087.0,294.0,2548.0,2453.0,386.0,2799.0,239.0,191.0,1717.0,1509.0,2099.0,911.0,1015.0,2590.0,85.0,3077.0,3264.0,3340.0,940.0,3114.0,146.0,1729.0,2498.0,309.0,1034.0,3484.0,2421.0,3438.0,2599.0,1827.0,1076.0,405.0,3461.0,3579.0,3813.0,1642.0,187.0,3238.0,3399.0,2005.0,3921.0,912.0,1840.0,2876.0,319.0,40.0,257.0,3287.0,880.0,844.0,754.0,1951.0,1874.0,1888.0,2241.0,2553.0,2203.0,1699.0,2297.0,550.0,2279.0,2886.0,3883.0,1549.0,2338.0,1922.0,452.0,3612.0,3144.0,1894.0,2026.0,2464.0,1049.0,2385.0,1824.0,2750.0,1185.0,2779.0,3902.0,2481.0,3580.0,2.0,2435.0,73.0,2956.0,3045.0,1012.0,1275.0,783.0,1473.0,2305.0,2049.0,512.0,2074.0,160.0,2720.0,1919.0,1814.0,3570.0,2531.0,375.0,3838.0,3549.0,2903.0,2778.0,507.0,1896.0,3451.0,2263.0,3199.0,3192.0,1671.0,2320.0,1515.0,438.0,363.0,341.0,3492.0,376.0,1562.0,2031.0,1295.0,1588.0,408.0,3405.0,1131.0,197.0,3894.0,307.0,1199.0,3268.0,447.0,186.0,808.0,2569.0,1961.0,1428.0,2540.0,1893.0,3284.0,1926.0,2873.0,2062.0,3624.0,1169.0,2513.0,3944.0,3128.0,575.0,1333.0,31.0,3595.0,3038.0,380.0,2696.0,3497.0,2342.0,2070.0,2130.0,1721.0,756.0,1932.0,1906.0,3662.0,935.0,3897.0,615.0,50.0,1105.0,956.0,1940.0,301.0,3330.0,2354.0,3406.0,84.0,99.0,3852.0,490.0,415.0,1797.0,1660.0,1811.0,1854.0,506.0,3625.0,406.0,3704.0,1094.0,2816.0,1163.0,2678.0,2122.0,1192.0,779.0,1836.0,2126.0,481.0,87.0,3577.0,2990.0,35.0,2629.0,3200.0,441.0,49.0,2864.0,1554.0,2784.0,3109.0,1499.0,346.0,577.0,1653.0,2202.0,3711.0,2616.0,1767.0,256.0,1117.0,1656.0,283.0,3584.0,2417.0,3721.0,2284.0,2978.0,3244.0,2042.0,397.0,2862.0,3454.0,1487.0,1582.0,2573.0,1018.0,1564.0,2568.0,2549.0,2844.0,1000.0,2054.0,2345.0,705.0,1669.0,1373.0,2048.0,429.0,3613.0,3663.0,1911.0,164.0,3924.0,949.0,420.0,3823.0,2570.0,359.0,92.0,2964.0,2434.0,179.0,217.0,1447.0,1719.0,2361.0,1943.0,148.0,1507.0,41.0,3895.0,1996.0,495.0,3949.0,2653.0,3103.0,2381.0,606.0,427.0,1413.0,379.0,3370.0,2379.0,1563.0,602.0,2711.0,2330.0,3447.0,3677.0,2293.0,2136.0,2642.0,1899.0,525.0,1050.0,2931.0,3138.0,3853.0,3363.0,3641.0,972.0,2962.0,3329.0,2358.0,169.0,2413.0,974.0,560.0,3206.0,3259.0,872.0,1995.0,1835.0,1496.0,3150.0,3805.0,2679.0,128.0,3518.0,2856.0,2925.0,483.0,543.0,462.0,2449.0,391.0,2586.0,2010.0,3393.0,1602.0,1755.0,3029.0,2494.0,2757.0,803.0,298.0,2500.0,1934.0,988.0,473.0,864.0,3007.0,2532.0,3261.0,2625.0,554.0,1631.0,2388.0,57.0,1167.0,2156.0,282.0,3456.0,249.0,382.0,2398.0,238.0,1798.0,1011.0,3217.0,1242.0,2953.0,2414.0,2550.0,1902.0,1084.0,2307.0,2128.0,131.0,2743.0,3055.0,1010.0,3290.0,1832.0,489.0,2630.0,3171.0,1949.0,245.0,3586.0,269.0,1954.0,2352.0,3343.0,2110.0,1579.0,129.0,3181.0,37.0,2021.0,2872.0,1344.0,3685.0,2257.0,2329.0,619.0,1354.0,1398.0,1312.0,659.0,2269.0,3428.0,804.0,2479.0,134.0,635.0,3866.0,2892.0,3950.0,2765.0,1444.0,422.0,2035.0,910.0,806.0,939.0,509.0,3859.0,1841.0,3738.0,173.0,1456.0,1756.0,1363.0,3041.0,3003.0,1427.0,3943.0,751.0,3445.0,986.0,2555.0,2614.0,997.0,192.0,3014.0,1561.0,1453.0,3874.0,136.0,2848.0,2722.0,2930.0,2992.0,3696.0,2103.0,3864.0,3707.0,776.0,1171.0,1384.0,558.0,1331.0,3501.0,639.0,2127.0,969.0,2559.0,3848.0,1139.0,166.0,1938.0,2823.0,2849.0,3664.0,1829.0,1107.0,2466.0,356.0,1812.0,1082.0,3194.0,185.0,1909.0,1349.0,2558.0,54.0,1316.0,2503.0,2426.0,2717.0,2309.0,3204.0,114.0,3881.0,2051.0,2219.0,476.0,2561.0,869.0,3843.0,934.0,2704.0,1709.0,149.0,1762.0,3667.0,371.0,2090.0,609.0,1112.0,291.0,3058.0,1831.0,2227.0,627.0,3197.0,499.0,2315.0,879.0,2439.0,621.0,389.0,3222.0,2768.0,3108.0,2517.0,3540.0,72.0,65.0,1477.0,3126.0,3215.0,594.0,1987.0,3423.0,1114.0,3833.0,1421.0,3502.0,1372.0,449.0,1822.0,2998.0,631.0,2874.0,2833.0,2974.0,3787.0,1672.0,2577.0,834.0,2047.0,2299.0,38.0,2180.0,807.0,3043.0,3665.0,1002.0,976.0,308.0,929.0,603.0,903.0,2304.0,3616.0,220.0,571.0,688.0,444.0,1056.0,3642.0,3390.0,1933.0,662.0,3136.0,960.0,1460.0,2587.0,2340.0,2797.0,3495.0,254.0,519.0,1232.0,2197.0,1211.0,767.0,3288.0,2697.0,596.0,3352.0,3892.0,984.0,2650.0,3446.0,320.0,435.0,769.0,3345.0,470.0,2884.0,787.0,1116.0,1730.0,612.0,3245.0,2516.0,2022.0,1446.0,411.0,1661.0,1546.0,278.0,3250.0,1523.0,2258.0,2934.0,1570.0,102.0,862.0,64.0,2445.0,746.0,798.0,1644.0,2854.0,2950.0,3550.0,2655.0,2562.0,2298.0,985.0,2067.0,640.0,2056.0,2147.0,982.0,887.0,1575.0,396.0,55.0,1614.0,212.0,3162.0,2079.0,42.0,2158.0,7.0,1685.0,559.0,1026.0,1572.0,3906.0,2608.0,1202.0,1442.0,2044.0,3824.0,3475.0,1556.0,1580.0,1720.0,1220.0,2727.0,1850.0,3027.0,2373.0,687.0,3010.0,833.0,1759.0,501.0,1659.0,124.0,1216.0,3293.0,1508.0,1815.0,1429.0,2800.0,2855.0,1857.0,139.0,2177.0,2063.0,3746.0,392.0,3542.0,189.0,2226.0,3913.0,156.0,18.0,475.0,2891.0,3854.0,1328.0,1754.0,3493.0,2683.0,2066.0,3028.0,2487.0,2162.0,2285.0,1881.0,3209.0,2949.0,2937.0,3119.0,2008.0,3862.0,3090.0,966.0,1593.0,1311.0,802.0,2831.0,1383.0,1870.0,2551.0,634.0,2440.0,407.0,2631.0,586.0,3480.0,1944.0,2989.0,3657.0,3858.0,1525.0,2695.0,2182.0,901.0,1489.0,1921.0,2785.0,1352.0,946.0,1862.0,97.0,1226.0,656.0,3581.0,1622.0,3885.0,2756.0,1988.0,1565.0,201.0,1162.0,3166.0,2260.0,2380.0,2339.0,1555.0,1950.0,1151.0,994.0,3137.0,3353.0,1306.0,3796.0,3489.0,2576.0,1990.0,2685.0,3160.0,3024.0,68.0,2777.0,3886.0,1323.0,2928.0,1023.0,3931.0,2190.0,398.0,2619.0,1061.0,1067.0,2919.0,3631.0,698.0,3459.0,2495.0,2444.0,3609.0,3232.0,775.0,3164.0,2370.0,2392.0,3645.0,821.0,753.0,1313.0,652.0,3761.0,108.0,3413.0,3592.0,306.0,2963.0,86.0,1484.0,1666.0,2429.0,3496.0,3474.0,3335.0,3878.0,1181.0,1514.0,352.0,1667.0,2363.0,98.0,2172.0,1174.0,692.0,3472.0,3315.0,1662.0,2184.0,1324.0,2036.0,295.0,3308.0,1206.0,697.0,1898.0,1077.0,1636.0,2039.0,3218.0,2233.0,1805.0,2715.0,3165.0,3453.0,641.0,205.0,2783.0,106.0,2714.0,2375.0,3173.0,216.0,3223.0,876.0,2382.0,1495.0,174.0,251.0,1694.0,736.0,119.0,3748.0,3131.0,1321.0,3517.0,1483.0,618.0,264.0,608.0,1005.0,1485.0,1347.0,498.0,3219.0,2766.0,1243.0,2789.0,2807.0,2069.0,25.0,347.0,90.0,2699.0,2251.0,1796.0,3757.0,3303.0,335.0,1792.0,1522.0,3414.0,3559.0,2814.0,2065.0,1937.0,2709.0,2860.0,2368.0,1833.0,1336.0,3598.0,549.0,2059.0,2465.0,1668.0,600.0,3715.0,2933.0,678.0,1845.0,2752.0,2075.0,1595.0,494.0,240.0,2790.0,645.0,3781.0,2018.0,1930.0,421.0,726.0,2273.0,62.0,1968.0,2120.0,2710.0,824.0,2477.0,866.0,712.0,1552.0,3841.0,219.0,418.0,1782.0,3050.0,2262.0,234.0,1566.0,2157.0,1749.0,2938.0,3064.0,243.0,3225.0,2087.0,3153.0,3907.0,3856.0,3829.0,3815.0,3750.0,3656.0,3650.0,3630.0,3607.0,3589.0,3583.0,3582.0,3561.0,3560.0,3558.0,3541.0,3530.0,3455.0,3411.0,3383.0,3382.0,3369.0,3356.0,3348.0,3332.0,3323.0,3279.0,3278.0,3234.0,3231.0,3227.0,3226.0,3195.0,3193.0,3191.0,3170.0,3080.0,3059.0,3023.0,3009.0,2980.0,2958.0,2957.0,2954.0,2910.0,2909.0,2845.0,2838.0,2832.0,2703.0,2698.0,2684.0,2680.0,2604.0,2603.0,2601.0,2595.0,2588.0,2564.0,2547.0,2543.0,2508.0,2489.0,2438.0,2319.0,2274.0,2270.0,2230.0,2229.0,2228.0,2225.0,2224.0,2222.0,2220.0,2216.0,2199.0,2198.0,2030.0,1868.0,1847.0,1843.0,1838.0,1828.0,1823.0,1819.0,1818.0,1813.0,1808.0,1803.0,1802.0,1800.0,1790.0,1789.0,1786.0,1781.0,1778.0,1776.0,1775.0,1774.0,1768.0,1766.0,1765.0,1763.0,1761.0,1757.0,1751.0,1745.0,1742.0,1740.0,1738.0,1737.0,1736.0,1723.0,1716.0,1712.0,1710.0,1708.0,1706.0,1705.0,1700.0,1698.0,1697.0,1691.0,1638.0,1637.0,1634.0,1628.0,1618.0,1607.0,1578.0,1577.0,1576.0,1568.0,1560.0,1559.0,1557.0,1540.0,1536.0,1530.0,1524.0,1521.0,1512.0,1506.0,1505.0,1492.0,1491.0,1481.0,1478.0,1469.0,1467.0,1462.0,1452.0,1451.0,1448.0,1443.0,1435.0,1434.0,1424.0,1418.0,1403.0,1402.0,1400.0,1368.0,1338.0,1319.0,1318.0,1314.0,1309.0,1308.0,1239.0,1229.0,1195.0,1182.0,1166.0,1159.0,1158.0,1157.0,1156.0,1155.0,1146.0,1143.0,1141.0,1140.0,1137.0,1122.0,1110.0,1109.0,1108.0,1106.0,1075.0,1074.0,1072.0,1065.0,1052.0,1048.0,1045.0,1001.0,995.0,983.0,979.0,894.0,890.0,883.0,873.0,871.0,857.0,856.0,855.0,845.0,825.0,822.0,819.0,817.0,816.0,812.0,797.0,795.0,794.0,777.0,773.0,772.0,770.0,768.0,752.0,740.0,739.0,738.0,727.0,723.0,721.0,713.0,699.0,693.0,689.0,686.0,683.0,677.0,676.0,675.0,654.0,646.0,636.0,629.0,625.0,624.0,622.0,620.0,604.0,591.0,404.0,403.0,400.0,399.0,395.0,323.0,286.0,285.0,284.0,221.0,143.0,127.0,115.0,109.0,91.0,51.0,1359.0,973.0,2108.0,3600.0,2859.0,502.0,1240.0,1707.0,1332.0,353.0,2798.0,1464.0,2637.0,2729.0,2461.0,1270.0,923.0,2159.0,3674.0,745.0,896.0,3258.0,80.0,147.0,3439.0,2941.0,229.0,836.0,1501.0,3637.0,714.0,2982.0,2947.0,2643.0,1935.0,3611.0,1212.0,2661.0,1343.0,2893.0,1310.0,2364.0,2984.0,1258.0,3253.0,1526.0,1809.0,2313.0,2552.0,964.0,2906.0,451.0,3419.0,2493.0,250.0,3486.0,374.0,2240.0,2527.0,167.0,213.0,472.0,2579.0,135.0,1419.0,3899.0,3945.0,853.0,3460.0,521.0,1217.0,814.0,2771.0,3378.0,2880.0,3376.0,3202.0,2217.0,2213.0,975.0,843.0,732.0,1993.0,1430.0,316.0,638.0,865.0,583.0,3336.0,868.0,1724.0,132.0,1613.0,2218.0,1293.0,993.0,3722.0,2519.0,3291.0,3797.0,2254.0,3904.0,1360.0,1863.0,1773.0,2636.0,1440.0,3065.0,258.0,649.0,2583.0,2386.0,1133.0,93.0,3237.0,3373.0,3621.0,3312.0,1386.0,1115.0,142.0,2812.0,1511.0,774.0,717.0,2420.0,3130.0,790.0,3001.0,2850.0,1820.0,2556.0,3277.0,3057.0,3666.0,2474.0,1877.0,3644.0,1651.0,2592.0,1165.0,3488.0,3888.0,771.0,730.0,1392.0,200.0,2879.0,226.0,991.0,3828.0,2895.0,1364.0,2308.0,3317.0,3941.0,1873.0,3597.0,2131.0,3403.0,962.0,3321.0,3220.0,2742.0,2484.0,2277.0,2235.0,1852.0,1630.0,1145.0,1118.0,859.0,651.0,644.0,579.0,2214.0,1070.0,763.0,826.0,1486.0,3574.0,2713.0,183.0,1520.0,2428.0,1714.0,3440.0,3297.0,2152.0,576.0,515.0,3236.0,77.0,1101.0,1905.0,453.0,344.0,2463.0,658.0,2578.0,1619.0,1470.0,1123.0,3085.0,2539.0,2996.0,1989.0,1224.0,1055.0,123.0,3151.0,311.0,1339.0,3359.0,1799.0,3070.0,1548.0,3782.0,895.0,1152.0,574.0,402.0,2749.0,2563.0,2758.0,3229.0,701.0,584.0,530.0,138.0,467.0,2242.0,3887.0,3243.0,3458.0,3040.0,2163.0,2135.0,1134.0,2281.0,3158.0,3437.0,660.0,657.0,1315.0,582.0,3026.0,2584.0,1908.0,1764.0,601.0,3653.0,508.0,1558.0,3485.0,1119.0,3283.0,67.0,718.0,580.0,2955.0,827.0,614.0,312.0,428.0,1189.0,2649.0,3463.0,1649.0,2932.0,3337.0,1816.0,120.0,2343.0,2716.0,3522.0,3772.0,3464.0,2292.0,1872.0,2533.0,3242.0,2865.0,578.0,655.0,2730.0,3687.0,820.0,884.0,2138.0,2191.0,3443.0,706.0,823.0,1590.0,3305.0,1795.0,2736.0,2033.0,2811.0,1125.0,1124.0,2776.0,792.0,1510.0,1102.0,642.0,1878.0,385.0,981.0,1969.0,2825.0,958.0,3743.0,789.0,3314.0,3212.0,488.0,3054.0,977.0,672.0,3404.0,1466.0,2821.0,1039.0,3548.0,3891.0,23.0,2674.0,3333.0,2310.0,1770.0,2124.0,1675.0,2176.0,288.0,1471.0,3207.0,2452.0,3377.0,3778.0,2735.0,690.0,684.0,2129.0,758.0,2794.0,401.0,2534.0,130.0,3533.0,1455.0,1991.0,633.0,75.0,3876.0,961.0,791.0,96.0,3172.0,190.0,3482.0,3292.0,1907.0,3647.0,2708.0,3318.0,2239.0,3184.0,666.0,757.0,2691.0,1553.0,2397.0,3900.0,3084.0,3837.0,782.0,3389.0,3889.0,1825.0,1681.0,3751.0,3779.0,1534.0,3381.0,2582.0,2632.0,454.0,1071.0,1004.0,2415.0,3736.0,2290.0,682.0,3442.0,2486.0,1234.0,3434.0,3205.0,1433.0,3735.0,545.0,3433.0,1502.0,2204.0,1864.0,2611.0,3216.0,696.0,33.0,3344.0,1817.0,1062.0,1647.0,3056.0,1887.0,2166.0,867.0,227.0,2016.0,310.0,2658.0,1839.0,3235.0,1664.0,1168.0,19.0,2071.0,3096.0,244.0,3182.0,573.0,1528.0,2032.0,2509.0,3850.0,3905.0,3627.0,3800.0,809.0,3342.0,3588.0,3346.0,598.0,1538.0,1849.0,1121.0,1098.0,2623.0,2394.0,2878.0,847.0,2265.0,2999.0,2223.0,2602.0,2221.0,2858.0,3575.0]

###################################

## For Flixster
#
# L_m = [292.0,487.0,727.0,790.0,700.0,238.0,263.0,215.0,723.0,443.0,501.0,365.0,557.0,691.0,821.0,468.0,565.0,216.0,224.0,350.0,37.0,746.0,260.0,655.0,366.0,302.0,453.0,414.0,210.0,763.0,563.0,1852.0,229.0,348.0,258.0,17.0,693.0,155.0,371.0,249.0,237.0,676.0,508.0,48.0,809.0,789.0,490.0,103.0,304.0,1862.0,318.0,436.0,690.0,469.0,309.0,1136.0,376.0,207.0,1346.0,394.0,567.0,337.0,1357.0,483.0,471.0,2477.0,416.0,930.0,797.0,559.0,2448.0,349.0,95.0,447.0,2445.0,1622.0,645.0,808.0,399.0,250.0,375.0,2720.0,899.0,395.0,905.0,919.0,409.0,461.0,844.0,931.0,552.0,996.0,590.0,877.0,703.0,1620.0,374.0,141.0,449.0,425.0,199.0,341.0,50.0,64.0,444.0,718.0,915.0,505.0,685.0,546.0,803.0,1228.0,140.0,80.0,2691.0,895.0,2685.0,1900.0,593.0,1128.0,1493.0,426.0,1350.0,1341.0,392.0,1475.0,1506.0,692.0,2822.0,267.0,476.0,2576.0,2442.0,1624.0,2607.0,321.0,297.0,1369.0,750.0,1876.0,362.0,2695.0,90.0,2598.0,530.0,486.0,171.0,298.0,147.0,148.0,1554.0,73.0,901.0,1242.0,762.0,232.0,683.0,2011.0,403.0,446.0,729.0,1895.0,1100.0,2132.0,43.0,832.0,798.0,404.0,2650.0,764.0,1529.0,882.0,1057.0,667.0,368.0,382.0,547.0,2597.0,203.0,333.0,639.0,1311.0,1374.0,2594.0,280.0,168.0,1427.0,11.0,65.0,2590.0,2367.0,927.0,114.0,109.0,18.0,158.0,598.0,2444.0,1381.0,608.0,2471.0,143.0,2492.0,86.0,1772.0,2591.0,2204.0,213.0,315.0,1255.0,101.0,438.0,512.0,369.0,2108.0,929.0,2307.0,479.0,1765.0,60.0,548.0,1177.0,191.0,715.0,772.0,92.0,2726.0,1576.0,1487.0,132.0,831.0,401.0,123.0,431.0,2051.0,435.0,276.0,2723.0,2491.0,1312.0,495.0,1173.0,284.0,1349.0,2246.0,270.0,594.0,2536.0,2026.0,2022.0,61.0,1834.0,1801.0,492.0,1109.0,1605.0,251.0,1735.0,1528.0,2565.0,561.0,373.0,1721.0,725.0,858.0,2436.0,464.0,2044.0,1033.0,1256.0,1492.0,356.0,108.0,234.0,938.0,2213.0,2656.0,124.0,872.0,518.0,1618.0,74.0,2258.0,765.0,2219.0,307.0,1967.0,1436.0,485.0,2320.0,558.0,656.0,1853.0,1053.0,2694.0,1541.0,1904.0,63.0,719.0,1338.0,303.0,2054.0,527.0,935.0,805.0,1028.0,942.0,2560.0,1501.0,1186.0,955.0,1936.0,1812.0,1950.0,2127.0,2434.0,1737.0,1850.0,437.0,330.0,282.0,336.0,424.0,379.0,1406.0,2234.0,345.0,2639.0,306.0,2447.0,2589.0,1867.0,2289.0,2684.0,1395.0,1847.0,1484.0,1316.0,1694.0,526.0,2672.0,206.0,1134.0,1822.0,2808.0,85.0,724.0,806.0,615.0,188.0,898.0,301.0,118.0,1644.0,1582.0,710.0,82.0,2439.0,432.0,285.0,1380.0,2519.0,1626.0,2702.0,54.0,165.0,1609.0,1523.0,682.0,819.0,2636.0,1239.0,1863.0,900.0,183.0,1796.0,440.0,1358.0,347.0,327.0,1636.0,1776.0,278.0,807.0,1425.0,2485.0,75.0,731.0,580.0,2253.0,2032.0,1055.0,1351.0,156.0,1875.0,1281.0,1121.0,2388.0,1683.0,1135.0,878.0,2378.0,462.0,2134.0,194.0,55.0,830.0,1851.0,625.0,2697.0,1106.0,187.0,1435.0,1556.0,1649.0,2165.0,2686.0,1969.0,1947.0,488.0,922.0,879.0,1778.0,286.0,480.0,768.0,550.0,2524.0,1522.0,2193.0,472.0,856.0,1808.0,622.0,1983.0,1387.0,1512.0,1409.0,541.0,459.0,857.0,2052.0,780.0,1645.0,386.0,1431.0,236.0,370.0,1469.0,120.0,2214.0,2282.0,24.0,2545.0,2382.0,290.0,2567.0,1993.0,2457.0,1191.0,1869.0,1285.0,383.0,1725.0,1166.0,1606.0,305.0,166.0,1590.0,713.0,800.0,429.0,1982.0,44.0,1489.0,1771.0,578.0,193.0,1391.0,1763.0,928.0,2009.0,2389.0,1245.0,2405.0,367.0,2074.0,1353.0,1651.0,2006.0,2757.0,2005.0,62.0,1359.0,42.0,570.0,1418.0,744.0,936.0,1344.0,1764.0,2043.0,1234.0,2174.0,174.0,2599.0,921.0,2454.0,1972.0,2604.0,1902.0,119.0,555.0,178.0,2654.0,679.0,2310.0,1294.0,2544.0,489.0,1096.0,227.0,1577.0,125.0,804.0,2273.0,2719.0,1920.0,1373.0,2527.0,1308.0,1914.0,1671.0,1365.0,504.0,1841.0,1303.0,2555.0,1814.0,441.0,2021.0,2133.0,2215.0,521.0,586.0,1318.0,1773.0,1558.0,1580.0,1768.0,2119.0,551.0,186.0,1525.0,89.0,2418.0,1149.0,1709.0,360.0,1526.0,1329.0,1032.0,1190.0,406.0,1524.0,2010.0,2559.0,2647.0,1722.0,545.0,2373.0,841.0,1268.0,2329.0,2581.0,648.0,1942.0,2430.0,916.0,2732.0,562.0,2055.0,2739.0,151.0,1009.0,2262.0,240.0,2768.0,503.0,2645.0,2665.0,1463.0,2064.0,1123.0,81.0,1401.0,417.0,902.0,1710.0,1408.0,1542.0,2783.0,576.0,1887.0,473.0,2774.0,991.0,268.0,2059.0,1572.0,1608.0,2704.0,1376.0,1631.0,1334.0,2625.0,1828.0,1840.0,716.0,1185.0,1581.0,2130.0,668.0,2255.0,2593.0,2239.0,1866.0,1858.0,924.0,1229.0,474.0,758.0,2314.0,2585.0,2252.0,1087.0,2463.0,592.0,2186.0,748.0,1926.0,2106.0,248.0,791.0,1510.0,2648.0,478.0,2274.0,2125.0,2394.0,1933.0,1152.0,340.0,1964.0,343.0,1368.0,1630.0,1424.0,664.0,2392.0,2818.0,1103.0,1968.0,69.0,2243.0,1945.0,2614.0,153.0,1111.0,1826.0,1584.0,2192.0,2003.0,707.0,2627.0,2035.0,1407.0,1732.0,463.0,161.0,1396.0,455.0,2018.0,2169.0,445.0,1378.0,1602.0,2816.0,189.0,415.0,2194.0,981.0,351.0,1050.0,2633.0,1453.0,640.0,68.0,975.0,2562.0,584.0,2107.0,1819.0,164.0,2530.0,1845.0,1877.0,848.0,1703.0,2668.0,1113.0,2602.0,378.0,1464.0,2767.0,2578.0,1901.0,281.0,880.0,1648.0,1485.0,2102.0,851.0,2634.0,1830.0,254.0,152.0,2759.0,1717.0,1414.0,1998.0,1356.0,252.0,1046.0,2266.0,1211.0,214.0,823.0,708.0,2655.0,2566.0,952.0,770.0,1958.0,2027.0,2111.0,2443.0,1790.0,1692.0,1757.0,2158.0,1916.0,1779.0,2080.0,2024.0,2669.0,1471.0,2049.0,220.0,1744.0,1347.0,2456.0,358.0,1623.0,87.0,1195.0,2118.0,596.0,346.0,2362.0,1446.0,2481.0,2605.0,1843.0,1497.0,2084.0,2455.0,2277.0,312.0,525.0,1552.0,1274.0,2643.0,411.0,2706.0,1613.0,897.0,1340.0,1751.0,2296.0,1324.0,162.0,2004.0,1856.0,225.0,638.0,1076.0,2183.0,2221.0,72.0,1687.0,1210.0,391.0,2212.0,2467.0,1518.0,1088.0,1214.0,2689.0,1723.0,217.0,589.0,1029.0,2191.0,946.0,966.0,1676.0,2302.0,154.0,1441.0,2701.0,1574.0,1480.0,2247.0,2177.0,982.0,450.0,842.0,2341.0,1802.0,9.0,741.0,2298.0,76.0,1569.0,2744.0,2601.0,465.0,1473.0,2493.0,2501.0,1138.0,2549.0,397.0,2093.0,1083.0,88.0,402.0,1238.0,2690.0,1400.0,1788.0,2269.0,735.0,816.0,104.0,1248.0,2628.0,2207.0,159.0,2352.0,2785.0,2236.0,914.0,2446.0,2336.0,1500.0,502.0,1905.0,2225.0,2408.0,122.0,1511.0,1986.0,277.0,1398.0,418.0,175.0,1278.0,1498.0,1394.0,2399.0,2497.0,361.0,1879.0,427.0,1729.0,2162.0,1240.0,1495.0,1575.0,2275.0,1980.0,1257.0,1254.0,2121.0,1419.0,222.0,77.0,1054.0,384.0,1226.0,1855.0,560.0,687.0,112.0,1216.0,28.0,799.0,1477.0,2749.0,2142.0,1443.0,554.0,1413.0,2148.0,1382.0,1944.0,2441.0,1227.0,1267.0,1708.0,2267.0,1355.0,2413.0,2577.0,2209.0,1886.0,2160.0,571.0,907.0,1686.0,1712.0,1272.0,2313.0,702.0,1604.0,2384.0,172.0,2651.0,2072.0,1361.0,2066.0,1189.0,180.0,1978.0,2317.0,1912.0,316.0,283.0,937.0,2638.0,2190.0,1325.0,1483.0,255.0,1601.0,481.0,2265.0,2045.0,749.0,230.0,2770.0,1258.0,2525.0,1913.0,888.0,1315.0,1815.0,454.0,2584.0,1393.0,1342.0,1827.0,2600.0,1736.0,976.0,1056.0,2105.0,1126.0,709.0,170.0,1793.0,1745.0,2698.0,144.0,2309.0,412.0,2743.0,2094.0,2374.0,1652.0,1996.0,1371.0,1741.0,1438.0,2295.0,2437.0,2136.0,2409.0,2603.0,2257.0,1295.0,2419.0,1707.0,2256.0,1437.0,1502.0,2721.0,2000.0,1198.0,1873.0,2122.0,660.0,2631.0,1929.0,1938.0,1531.0,2235.0,127.0,1388.0,2056.0,953.0,2188.0,2023.0,1760.0,1675.0,1921.0,2229.0,344.0,1726.0,1007.0,2474.0,2755.0,308.0,136.0,932.0,1586.0,1829.0,2086.0,2754.0,1906.0,1488.0,1172.0,131.0,1383.0,2397.0,363.0,1491.0,2306.0,2395.0,2552.0,2172.0,496.0,971.0,769.0,2404.0,1451.0,960.0,2248.0,2146.0,1074.0,1494.0,1813.0,747.0,1940.0,867.0,2569.0,2202.0,884.0,1169.0,2509.0,83.0,1490.0,300.0,2649.0,262.0,99.0,1544.0,883.0,2377.0,2571.0,2244.0,1065.0,2046.0,1789.0,1621.0,2713.0,2103.0,261.0,16.0,581.0,1961.0,1897.0,200.0,2128.0,2340.0,2688.0,801.0,434.0,2197.0,2401.0,2025.0,2683.0,1252.0,2641.0,149.0,2494.0,2386.0,1282.0,2687.0,1659.0,1147.0,2383.0,1165.0,1517.0,2047.0,1206.0,1095.0,1894.0,1896.0,612.0,1956.0,2123.0,887.0,1730.0,2076.0,1706.0,204.0,1720.0,2414.0,2804.0,121.0,2789.0,2081.0,2126.0,274.0,2338.0,1131.0,1118.0,1476.0,2173.0,2028.0,1444.0,1704.0,2805.0,1354.0,2564.0,2033.0,2155.0,1530.0,430.0,2327.0,1283.0,134.0,1068.0,2775.0,1565.0,12.0,1553.0,2264.0,2498.0,2230.0,2167.0,1270.0,992.0,1036.0,1059.0,1946.0,2422.0,2762.0,2070.0,2421.0,1301.0,498.0,2350.0,1999.0,2674.0,2211.0,2542.0,2728.0,2510.0,1617.0,289.0,2618.0,1465.0,1589.0,1225.0,421.0,197.0,773.0,2129.0,2677.0,2079.0,1949.0,1953.0,1599.0,2478.0,2344.0,466.0,2588.0,1962.0,1231.0,1327.0,1881.0,1738.0,2085.0,2371.0,142.0,1557.0,2363.0,654.0,1780.0,201.0,1591.0,1114.0,753.0,2312.0,1313.0,1888.0,422.0,1161.0,2831.0,196.0,2615.0,845.0,740.0,2573.0,173.0,2145.0,1579.0,334.0,458.0,97.0,1971.0,2366.0,2734.0,2782.0,1328.0,1004.0,2380.0,706.0,2100.0,2480.0,1610.0,1697.0,1762.0,652.0,2705.0,19.0,2801.0,2335.0,2067.0,2170.0,400.0,381.0,1092.0,2227.0,494.0,1077.0,1791.0,1922.0,630.0,2534.0,1990.0,113.0,2161.0,1170.0,2206.0,2742.0,2272.0,1047.0,983.0,2670.0,167.0,2288.0,587.0,2435.0,621.0,2291.0,2833.0,398.0,2515.0,1798.0,2520.0,2693.0,2699.0,1804.0,777.0,1250.0,1385.0,2164.0,822.0,2038.0,233.0,1573.0,1027.0,722.0,2402.0,1402.0,2529.0,2233.0,2788.0,1882.0,1410.0,517.0,1989.0,1209.0,106.0,1188.0,2099.0,2533.0,1939.0,2322.0,2372.0,629.0,2516.0,299.0,2238.0,1403.0,1496.0,1935.0,1117.0,1770.0,2396.0,2609.0,2015.0,2089.0,632.0,1619.0,1747.0,577.0,1462.0,1299.0,1653.0,2304.0,2403.0,634.0,2764.0,2251.0,2062.0,2550.0,988.0,1988.0,1563.0,2171.0,2323.0,2153.0,2387.0,2135.0,1578.0,1719.0,2518.0,452.0,1660.0,2364.0,666.0,1607.0,2368.0,1513.0,1752.0,2328.0,1817.0,2150.0,7.0,45.0,2029.0,2806.0,963.0,2181.0,2792.0,2263.0,2610.0,295.0,1072.0,2245.0,1655.0,2069.0,2554.0,1503.0,2325.0,1280.0,177.0,2718.0,1421.0,2820.0,1073.0,2830.0,2505.0,93.0,913.0,2285.0,2205.0,2802.0,1821.0,1781.0,354.0,1943.0,1051.0,2692.0,730.0,2727.0,1825.0,2031.0,1566.0,1130.0,1871.0,2166.0,2037.0,890.0,2671.0,1375.0,2117.0,2587.0,1854.0,273.0,1458.0,219.0,1670.0,1197.0,228.0,2540.0,1459.0,2787.0,506.0,2786.0,1749.0,1527.0,651.0,1041.0,1807.0,2400.0,2790.0,2115.0,2707.0,2417.0,2339.0,2551.0,2810.0,192.0,644.0,1176.0,1562.0,2065.0,408.0,266.0,2333.0,2522.0,2722.0,1750.0,94.0,1955.0,1019.0,1253.0,2700.0,2539.0,1430.0,2548.0,2748.0,1417.0,896.0,885.0,245.0,2063.0,2096.0,2092.0,2203.0,796.0,2008.0,1681.0,2741.0,246.0,1654.0,1450.0,1063.0,497.0,1872.0,256.0,2276.0,1880.0,185.0,1335.0,2308.0]
# L_f = [56.0,13.0,607.0,393.0,513.0,617.0,1090.0,869.0,721.0,732.0,533.0,529.0,582.0,825.0,697.0,597.0,21.0,1011.0,870.0,569.0,775.0,649.0,27.0,847.0,47.0,751.0,618.0,940.0,35.0,1024.0,711.0,674.0,314.0,616.0,396.0,760.0,658.0,650.0,886.0,945.0,745.0,595.0,637.0,67.0,788.0,40.0,271.0,833.0,795.0,846.0,659.0,1559.0,2.0,137.0,868.0,160.0,984.0,1891.0,2346.0,475.0,850.0,243.0,920.0,342.0,1091.0,34.0,31.0,1098.0,761.0,241.0,572.0,728.0,588.0,815.0,613.0,910.0,129.0,1831.0,840.0,5.0,568.0,235.0,272.0,918.0,1125.0,1682.0,1080.0,39.0,420.0,778.0,128.0,599.0,626.0,1140.0,1310.0,793.0,591.0,514.0,2176.0,782.0,433.0,2617.0,1348.0,1640.0,1105.0,861.0,33.0,516.0,1199.0,329.0,1331.0,326.0,1141.0,829.0,906.0,510.0,287.0,528.0,377.0,1187.0,1184.0,776.0,1650.0,1742.0,811.0,678.0,534.0,319.0,1008.0,1307.0,1084.0,694.0,1456.0,423.0,1297.0,1212.0,231.0,275.0,1180.0,1089.0,20.0,1685.0,2451.0,355.0,2311.0,1785.0,1468.0,909.0,958.0,956.0,53.0,456.0,29.0,912.0,661.0,944.0,836.0,2450.0,854.0,244.0,540.0,860.0,389.0,852.0,1208.0,862.0,2185.0,23.0,1261.0,893.0,1218.0,855.0,853.0,2479.0,544.0,1194.0,1273.0,1168.0,812.0,467.0,814.0,1305.0,754.0,686.0,405.0,1174.0,641.0,1673.0,605.0,1309.0,2504.0,1592.0,1959.0,1479.0,837.0,2385.0,1232.0,834.0,1832.0,1960.0,38.0,826.0,1075.0,1288.0,179.0,717.0,115.0,614.0,939.0,1455.0,317.0,964.0,1509.0,757.0,1889.0,388.0,781.0,1277.0,1235.0,226.0,677.0,1286.0,627.0,787.0,665.0,1551.0,269.0,1537.0,1262.0,1066.0,779.0,357.0,221.0,689.0,419.0,1265.0,1181.0,1560.0,1372.0,1637.0,1017.0,1643.0,1139.0,972.0,828.0,705.0,1674.0,1049.0,1000.0,1805.0,1244.0,1625.0,1612.0,482.0,198.0,1021.0,1690.0,2390.0,669.0,1898.0,794.0,726.0,1696.0,1151.0,1429.0,1434.0,766.0,1016.0,448.0,1705.0,332.0,212.0,926.0,1317.0,1992.0,520.0,2653.0,774.0,51.0,1182.0,1857.0,827.0,145.0,1129.0,1070.0,663.0,759.0,1364.0,1052.0,1290.0,71.0,1600.0,146.0,1003.0,2769.0,523.0,771.0,532.0,32.0,943.0,41.0,704.0,1915.0,752.0,993.0,1192.0,470.0,353.0,631.0,2330.0,2124.0,859.0,987.0,1001.0,714.0,911.0,1062.0,785.0,675.0,2334.0,2827.0,107.0,2261.0,1668.0,1025.0,2592.0,1641.0,294.0,1115.0,537.0,1167.0,1263.0,712.0,2771.0,2523.0,1101.0,733.0,2120.0,1040.0,1367.0,2500.0,1026.0,962.0,2823.0,1952.0,838.0,247.0,864.0,1314.0,979.0,1702.0,102.0,1693.0,737.0,59.0,1160.0,2582.0,934.0,428.0,1951.0,1748.0,619.0,1243.0,2753.0,1058.0,14.0,1363.0,1158.0,1339.0,1684.0,1910.0,2486.0,865.0,1154.0,3.0,891.0,1379.0,933.0,628.0,1179.0,1193.0,701.0,291.0,2449.0,2620.0,1661.0,2391.0,1213.0,2635.0,1266.0,646.0,950.0,1657.0,903.0,1107.0,1022.0,296.0,2426.0,994.0,949.0,642.0,2673.0,636.0,2154.0,1196.0,8.0,1549.0,2196.0,538.0,2737.0,1386.0,1811.0,738.0,139.0,1205.0,407.0,2814.0,1093.0,1870.0,2661.0,372.0,439.0,2538.0,1984.0,22.0,1405.0,1508.0,875.0,997.0,2483.0,2487.0,1071.0,623.0,999.0,2098.0,176.0,2058.0,624.0,1291.0,959.0,1700.0,1304.0,36.0,1082.0,1069.0,2745.0,1733.0,135.0,904.0,1839.0,600.0,1296.0,1691.0,1333.0,1044.0,863.0,30.0,1595.0,2679.0,1204.0,1775.0,1864.0,601.0,1930.0,1743.0,1023.0,2703.0,977.0,2657.0,1097.0,2696.0,2299.0,2621.0,1415.0,484.0,951.0,873.0,2157.0,70.0,734.0,1302.0,507.0,1820.0,499.0,989.0,2259.0,2226.0,1061.0,2580.0,767.0,1143.0,1739.0,2041.0,1330.0,2556.0,1783.0,1332.0,1223.0,606.0,866.0,2425.0,1893.0,941.0,1148.0,784.0,1627.0,1628.0,1809.0,1701.0,265.0,1156.0,2499.0,755.0,2048.0,1439.0,1787.0,1336.0,2453.0,1987.0,1844.0,1594.0,58.0,1215.0,2733.0,917.0,2570.0,662.0,1119.0,138.0,1457.0,2667.0,1908.0,2642.0,2379.0,1031.0,2381.0,2351.0,2357.0,1838.0,2459.0,1178.0,1006.0,157.0,954.0,1222.0,1287.0,2370.0,1064.0,1907.0,2147.0,442.0,2208.0,2511.0,1094.0,1755.0,1478.0,1247.0,2461.0,1230.0,1885.0,783.0,1202.0,1519.0,1035.0,2297.0,1150.0,1010.0,2271.0,1045.0,2765.0,46.0,2731.0,2189.0,1792.0,338.0,1570.0,2819.0,969.0,1104.0,1013.0,1120.0,1432.0,2007.0,519.0,2198.0,324.0,2711.0,1322.0,1598.0,2149.0,1155.0,2572.0,1634.0,1948.0,4.0,1777.0,1614.0,2676.0,2780.0,1666.0,1688.0,1442.0,2280.0,1975.0,2681.0,2053.0,1642.0,2464.0,1233.0,2200.0,1925.0,325.0,1163.0,1162.0,970.0,335.0,2660.0,1099.0,259.0,2537.0,1698.0,1766.0,1360.0,1689.0,1014.0,756.0,1533.0,2090.0,491.0,2143.0,1545.0,2101.0,1795.0,835.0,2116.0,2137.0,66.0,1656.0,720.0,1146.0,2231.0,279.0,1593.0,2423.0,1472.0,2736.0,1320.0,990.0,1039.0,2294.0,57.0,2659.0,1997.0,1507.0,2613.0,2543.0,2144.0,2462.0,1800.0,2512.0,1200.0,1713.0,2495.0,1837.0,1664.0,2750.0,2735.0,1276.0,328.0,1319.0,1823.0,1300.0,2644.0,2415.0,1159.0,603.0,1271.0,1718.0,6.0,1337.0,923.0,810.0,698.0,2199.0,1567.0,2110.0,390.0,1390.0,2433.0,1919.0,2502.0,1934.0,2406.0,1352.0,1797.0,980.0,1370.0,2411.0,2017.0,742.0,2424.0,539.0,320.0,575.0,110.0,310.0,1420.0,2113.0,1849.0,2626.0,1753.0,522.0,515.0,1611.0,2606.0,2747.0,111.0,1756.0,2287.0,974.0,2210.0,205.0,2354.0,2036.0,208.0,1699.0,1110.0,195.0,948.0,2541.0,2301.0,2361.0,1461.0,573.0,2260.0,2141.0,1966.0,1034.0,2708.0,1754.0,957.0,2488.0,1928.0,1389.0,1219.0,1.0,2640.0,1550.0,1275.0,2156.0,1848.0,2760.0,2712.0,2776.0,688.0,2716.0,2466.0,2095.0,1639.0,1224.0,1279.0,1981.0,1015.0,1774.0,1037.0,1715.0,1020.0,647.0,2558.0,1384.0,387.0,531.0,2517.0,1833.0,2793.0,1767.0,2784.0,1246.0,1298.0,813.0,385.0,79.0,1474.0,1924.0,1957.0,2220.0,1133.0,2326.0,10.0,1048.0,2068.0,2331.0,2305.0,973.0,1918.0,91.0,1217.0,2715.0,2662.0,242.0,1977.0,2460.0,1874.0,460.0,1724.0,181.0,894.0,1157.0,2469.0,323.0,1269.0,2242.0,1241.0,1571.0,2612.0,2777.0,542.0,2778.0,2365.0,2324.0,1635.0,2664.0,968.0,2337.0,2622.0,2109.0,1521.0,1108.0,1846.0,743.0,2020.0,1454.0,2440.0,553.0,1144.0,585.0,2195.0,2646.0,1284.0,2332.0,451.0,611.0,2175.0,2834.0,2680.0,2353.0,1974.0,1991.0,100.0,2222.0,1758.0,2608.0,1665.0,602.0,1532.0,1397.0,2791.0,1124.0,2087.0,2030.0,1859.0,2429.0,182.0,1903.0,1085.0,2795.0,2168.0,2279.0,1669.0,680.0,2547.0,2682.0,579.0,1629.0,1392.0,2016.0,2292.0,322.0,1449.0,2561.0,583.0,2284.0,2300.0,1481.0,264.0,2476.0,1667.0,1884.0,2675.0,609.0,1067.0,1486.0,1404.0,1145.0,1460.0,1646.0,339.0,1467.0,1803.0,1081.0,1555.0,1824.0,2616.0,2710.0,1979.0,457.0,2393.0,610.0,1326.0,2528.0,2809.0,2278.0,2039.0,1892.0,2678.0,2666.0,253.0,2012.0,1728.0,2563.0,1672.0,1588.0,2179.0,1251.0,549.0,1260.0,1954.0,1539.0,117.0,2503.0,2014.0,2807.0,1663.0,2484.0,2557.0,1734.0,1038.0,1002.0,2583.0,2290.0,2077.0,2184.0,2432.0,1899.0,1183.0,2568.0,493.0,2356.0,671.0,1366.0,1842.0,2223.0,2611.0,209.0,817.0,2794.0,1142.0,673.0,2781.0,2428.0,2427.0,133.0,410.0,2281.0,257.0,1259.0,2663.0,1249.0,2482.0,2532.0,2355.0,2652.0,2375.0,2575.0,2595.0,2729.0,1236.0,2224.0,1716.0,49.0,1932.0,2632.0,78.0,511.0,1633.0,2347.0,2152.0,566.0,116.0,1677.0,843.0,163.0,2407.0,1799.0,1810.0,1292.0,1931.0,2709.0,633.0,1616.0,1018.0,1836.0,2237.0,2624.0,1647.0,2140.0,2342.0,1883.0,1416.0,2345.0,1909.0,1482.0,818.0,1835.0,311.0,2812.0,1923.0,1264.0,2828.0,2752.0,2060.0,1678.0,293.0,1411.0,2078.0,1714.0,2452.0,1890.0,543.0,2303.0,1362.0,1171.0,1399.0,2349.0,889.0,681.0,1973.0,190.0,130.0,2151.0,2773.0,239.0,1440.0,2218.0,211.0,2369.0,150.0,1306.0,1030.0,2756.0,736.0,739.0,672.0,2431.0,2138.0,1412.0,1078.0,2420.0,2472.0,2535.0,2217.0,509.0,1112.0,1963.0,2763.0,1941.0,2835.0,2797.0,2438.0,2359.0,871.0,699.0,1878.0,978.0,2531.0,1060.0,1122.0,635.0,1917.0,985.0,849.0,2730.0,524.0,564.0,2319.0,839.0,786.0,1680.0,1538.0,1447.0,802.0,1638.0,1102.0,2270.0,1597.0,1536.0,1445.0,1540.0,105.0,2725.0,2811.0,1927.0,1499.0,2546.0,643.0,1516.0,1782.0,696.0,2506.0,947.0,331.0,1086.0,352.0,1237.0,413.0,1345.0,2348.0,1587.0,2758.0,1221.0,1994.0,892.0,2489.0,2751.0,2042.0,1079.0,2293.0,2496.0,2514.0,1937.0,2507.0,25.0,1132.0,2824.0,792.0,1470.0,2586.0,2316.0,2490.0,1740.0,26.0,2139.0,1759.0,2410.0,223.0,1504.0,2738.0,1806.0,2250.0,574.0,2513.0,1543.0,2187.0,1127.0,824.0,1561.0,620.0,500.0,2061.0,2001.0,2178.0,1548.0,1220.0,2803.0,1727.0,2268.0,96.0,1343.0,670.0,2798.0,1695.0,1116.0,1293.0,908.0,684.0,2216.0,1323.0,1970.0,2637.0,2619.0,986.0,1164.0,2057.0,1995.0,2318.0,2358.0,2112.0,2131.0,1911.0,2114.0,2596.0,15.0,1662.0,313.0,2815.0,2821.0,995.0,2817.0,2766.0,1207.0,364.0,967.0,820.0,2473.0,604.0,1153.0,1615.0,695.0,2832.0,1433.0,2083.0,2416.0,1043.0,1452.0,876.0,961.0,1005.0,2826.0,2082.0,2315.0,1514.0,1423.0,653.0,2398.0,1585.0,2040.0,1746.0,535.0,1976.0,2475.0,1786.0,2376.0,2091.0,1865.0,2241.0,477.0,2623.0,1603.0,1711.0,2724.0,1466.0,1860.0,98.0,2468.0,2553.0,2465.0,1816.0,2508.0,2630.0,1203.0,1568.0,2232.0,2050.0,2800.0,1632.0,2458.0,1761.0,2097.0,2088.0,925.0,2075.0,2658.0,1769.0,288.0,2019.0,2740.0,2779.0,1012.0,1428.0,1175.0,2360.0,2013.0,1546.0,1547.0,1868.0,184.0,1377.0,218.0,1658.0,2104.0,874.0,1505.0,2249.0,52.0,1289.0,1861.0,1818.0,2772.0,556.0,126.0,2746.0,84.0,2071.0,2283.0,2574.0,1965.0,1985.0,2343.0,2073.0,1583.0,1321.0,2002.0,2521.0,2286.0,1448.0,1534.0,2813.0,2254.0,1201.0,881.0,1422.0,2799.0,1564.0,998.0,1535.0,2034.0,2180.0,1679.0,1426.0,2159.0,1794.0,1137.0,2240.0,1731.0,380.0,202.0,2717.0,2163.0,169.0,359.0,1784.0,2629.0,536.0,2321.0,1515.0,2825.0,2761.0,2714.0,965.0,1596.0,2526.0,1520.0,2829.0,2796.0,2412.0,2228.0,1042.0,2182.0,657.0,2201.0,2579.0,2470.0,2308.0,1335.0,185.0,1880.0,2276.0,256.0,1872.0,497.0,1063.0,1450.0,1654.0,246.0,2741.0,1681.0,2008.0,796.0,2203.0,2092.0,2096.0,2063.0,245.0,885.0,896.0,1417.0,2748.0,2548.0,1430.0,2539.0,2700.0,1253.0,1019.0,1955.0,94.0,1750.0,2722.0,2522.0,2333.0,266.0,408.0,2065.0,1562.0,1176.0,644.0,192.0,2810.0,2551.0,2339.0,2417.0,2707.0,2115.0,2790.0,2400.0,1807.0,1041.0,651.0,1527.0,1749.0,2786.0,506.0,2787.0,1459.0,2540.0,228.0,1197.0,1670.0,219.0,1458.0,273.0,1854.0,2587.0,2117.0,1375.0,2671.0,890.0,2037.0,2166.0,1871.0,1130.0,1566.0,2031.0,1825.0,2727.0,730.0,2692.0,1051.0,1943.0,354.0,1781.0,1821.0,2802.0,2205.0,2285.0,913.0,93.0,2505.0,2830.0,1073.0,2820.0,1421.0,2718.0,177.0,1280.0,2325.0,1503.0,2554.0,2069.0,1655.0,2245.0,1072.0,295.0,2610.0,2263.0,2792.0,2181.0,963.0,2806.0,2029.0,45.0,7.0,2150.0,1817.0,2328.0,1752.0,1513.0,2368.0,1607.0,666.0,2364.0,1660.0,452.0,2518.0]

# For LastFM
from numpy import loadtxt
# Note to Me: Don't forget to +1 if I use this list in other places
# new_lm = [x+1 for x in L_m] new_lf = [x+1 for x in L_f]
L_m = loadtxt("lst_LFM_male.txt", comments="#", delimiter=",", unpack=False)
L_f = loadtxt("lst_LFM_female.txt", comments="#", delimiter=",", unpack=False)
def blurMe_1m():
    sample_mode = list(['random', 'sampled', 'greedy'])[2]
    rating_mode = list(['highest', 'avg', 'pred'])[1]
    top = -1
    p = 0.05
    notice_factor = 2
    dataset = ['ML', 'Fx', 'LFM', 'Li'][1]
    if dataset == 'ML': # load_user_item_matrix_1m_all
        X = MD.load_user_item_matrix_1m_trainingSet() #load_user_item_matrix_1m_trainingSet MD.load_user_item_matrix_1m_all()  # max_user=max_user, max_item=max_item)
        T = MD.load_gender_vector_1m()  # max_user=max_user)
        X_test = MD.load_user_item_matrix_1m_testSet()
    elif dataset == 'Fx':
        """import FlixsterData as FD
        X, T, _ = FD.load_flixster_data_subset()"""
        ###X, T = FD.load_flixster_data()
        #X1 = FD.load_user_item_matrix_FD_All ()
        #X, T = FD.load_flixster_data()
        #T = np.loadtxt('FX_Users6000_Gender.txt', dtype=int)
        import FlixsterDataSub as FDS
        X = FDS.load_user_item_matrix_FX_All()
        # X = FDS.load_user_item_matrix_FX_TrainingSet()
        X_test = FDS.load_user_item_matrix_FX_Test()
        T = FDS.load_gender_vector_FX()

    elif dataset == 'LFM':
        import LastFMData as LFM
        # X = LFM.load_user_item_matrix_lfm_Train()# LFM.load_user_item_matrix_lfm_All()
        X = LFM.load_user_item_matrix_lfm_All()  # load_user_item_matrix_lfm_Train LFM.load_user_item_matrix_lfm_All()
        T = LFM.load_gender_vector_lfm()
        X_test = LFM.load_user_item_matrix_lfm_Test()
    else:
        import LibimSeTiData as LD
        X, T, _ = LD.load_libimseti_data2()
    # X = Utils.normalizze(X)

    # print(X)
    """
    avg_ratings = np.zeros(shape=X.shape[0])
    for index, user in enumerate(X):
        ratings = []
        for rating in user:
            if rating > 0:
                ratings.append(rating)
        if len(ratings) == 0:
            avg_ratings[index] = 0
        else:
            avg_ratings[index] = np.average(ratings)"""

    avg_ratings = np.zeros(shape=X.shape[1])
    initial_count = np.zeros(shape=X.shape[1])
    for item_id in range(X.shape[1]):
        ratings = []
        for rating in X[:, item_id]:
            if rating > 0:
                ratings.append(rating)
        if len(ratings) == 0:
            avg_ratings[item_id] = 0
        else:
            avg_ratings[item_id] = np.average(ratings)
        initial_count[item_id] = len(ratings)
    max_count = initial_count * notice_factor

    """ AVERAGE ACROSS MOVIE
    avg_ratings = np.zeros(shape=X.shape[1])
    for item_id in range(X.shape[1]):
        ratings = []
        for rating in X[:, item_id]:
            if rating > 0:
                ratings.append(rating)
        if len(ratings) == 0:
            avg_ratings[item_id] = 0
        else:
            avg_ratings[item_id] = np.average(ratings)
    """

    """
    # 1: get the set of most correlated movies, L_f and L_m:
    X_train, T_train = X[0:int(0.8 * len(X))], T[0:int(0.8 * len(X))]
    X_test, T_test = X[int(0.8 * len(X)):], T[int(0.8 * len(X)):]
    print("lists")
    from sklearn.model_selection import StratifiedKFold
    from sklearn.linear_model import LogisticRegression

    cv = StratifiedKFold(n_splits=10)
    coefs = []
    avg_coefs = np.zeros(shape=(len(X_train[1]),))

    random_state = np.random.RandomState(0)
    for train, test in cv.split(X_train, T_train):
        x, t = X_train[train], T_train[train]
        # grid = {"C":[0.001,.009,0.01,.09,1,5,10,25], "penalty": ["l1", "l2"]}  # l1 lasso l2 ridge
        model = LogisticRegression(penalty='l2', random_state=random_state)
        # clf = LogisticRegression()
        # model = GridSearchCV(clf, param_grid = grid)

        model.fit(x, t)
        # rank the coefs:
        ranks = ss.rankdata(model.coef_[0])
        coefs.append(ranks)
        #print(len(model.coef_[0]),len(X_train[0]))
        avg_coefs += model.coef_[0]

    coefs = np.average(coefs, axis=0)
    coefs = [[coefs[i], i+1, avg_coefs[i]] for i in range(len(coefs))]
    coefs = np.asarray(list(sorted(coefs)))

    if top == -1:
        values = coefs[:,2]
        var_val = np.min(np.abs(values))
        index_zero = np.where(np.abs(values) == np.min(np.abs(values)))
        top_male = index_zero[0][0]
        top_female = index_zero[0][-1]
        L_m = coefs[:top_male, 1]
        R_m = 885 - coefs[:top_male, 0] # 3952 2835
        C_m = np.abs(coefs[:top_male, 2])
        L_f = coefs[coefs.shape[0] - top_female:, 1]
        L_f = list(reversed(L_f))
        R_f = coefs[coefs.shape[0] - top_female:, 0]
        R_f = list(reversed(R_f))
        C_f = coefs[coefs.shape[0] - top_female:, 2]
        C_f = list(reversed(np.abs(C_f)))

    else:
        L_m = coefs[:top, 1]
        R_m = 885 -coefs[:top, 0] # 3952
        C_m = np.abs(coefs[:top, 2])
        L_f = coefs[coefs.shape[0]-top:, 1]
        L_f = list(reversed(L_f))
        R_f = coefs[coefs.shape[0]-top:, 0]
        R_f = list(reversed(R_f))
        C_f = coefs[coefs.shape[0]-top:, 2]
        C_f = list(reversed(np.abs(C_f)))

    #print(R_f)
    #L_m = L_m[:500]
    #L_f= L_f[:500]
    print("L_f: ", L_f )
    print("L_m: ", L_m)
    print(len(L_f))
    L_ff = L_f.copy()
    np.savetxt("lst_LFM_female.txt", L_f, delimiter=",", newline="\n", fmt="%s")
    # L_ff = pd.DataFrame(L_ff)
    # L_ff.to_csv('L_ff_BlurMe_lfm_Normalized.csv', index = False)
    print("------")
    print(len(L_m))
    L_mm = L_m.copy()
    np.savetxt("lst_LFM_male.txt", L_m, delimiter=",", newline="\n", fmt="%s")

    # L_mm = pd.DataFrame(L_mm)
    # L_mm.to_csv('L_mm_BlurMe_lfm_Normalized.csv', index = False)
    """

    """
    id_index, index_id = MD.load_movie_id_index_dict()
    movies = []
    with open("ml-1m/movies.dat", 'r') as f:
        for line in f.readlines():
            movies.append(line.replace("\n", ""))

    for index, val in enumerate(L_m[0:10]):
        print(index, movies[id_index[int(val)]], C_m[index])
    for index, val in enumerate(L_f[0:10]):
        print(index, movies[id_index[int(val)]], C_f[index])

    
    movie_dict = MD.load_movie_id_dictionary_1m()
    print("males")
    for id in L_m:
        print(movie_dict[int(id)])

    print("females")
    for id in L_f:
        print(movie_dict[int(id)])"""

    # from numpy import loadtxt
    # L_m = loadtxt("lst_LFM_male.txt", comments="#", delimiter=",", unpack=False)
    # L_f = loadtxt("lst_LFM_female.txt", comments="#", delimiter=",", unpack=False)


    print("obfuscation")
    # Now, where we have the two lists, we can start obfuscating the data:
    #X = MD.load_user_item_matrix_1m()
    X_obf = np.copy(X)

    #X = Utils.normalize(X)
    #X_obf = Utils.normalize(X_obf)
    prob_m = []#[p / sum(C_m) for p in C_m]
    prob_f = []#[p / sum(C_f) for p in C_f]
    print("obfuscation")
    for index, user in enumerate(X):
        print(index)
        k = 0
        for rating in user:
            if rating > 0:
                k += 1
        k *= p
        greedy_index = 0
        #print(k)
        if T[index] == 1:
            added = 0
            safety_counter = 0
            while added < k and safety_counter < 100:
                # select a random movie:
                if sample_mode == 'random':
                    movie_id = L_m[np.random.randint(0, len(L_m))]
                elif sample_mode == 'sampled':
                    movie_id = L_m[np.random.choice(range(len(L_m)), p=prob_m)]
                elif sample_mode == 'greedy':
                    movie_id = L_m[greedy_index]
                    greedy_index += 1
                    if greedy_index >= len(L_m):
                        safety_counter = 100
                rating_count = sum([1 if x > 0 else 0 for x in X_obf[:, int(movie_id) - 1]])
                if rating_count > max_count[int(movie_id) - 1]:
                    continue
                if X_obf[index, int(movie_id)-1] == 0:# and X_test [index, int(movie_id) - 1] ==0:
                    if rating_mode == 'higest':
                        X_obf[index, int(movie_id) - 1] = 5
                    elif rating_mode == 'avg':
                        X_obf[index, int(movie_id) - 1] = avg_ratings[int((movie_id) - 1) ] # avg_ratings[int(index)]
                    added += 1
                safety_counter += 1
        elif T[index] == 0:
            added = 0
            safety_counter = 0
            while added < k and safety_counter < 100:
                # select a random movie:
                if sample_mode == 'random':
                    movie_id = L_f[np.random.randint(0, len(L_f))]
                elif sample_mode == 'sampled':
                    movie_id = L_f[np.random.choice(range(len(L_f)), p=prob_f)]
                elif sample_mode == 'greedy':
                    movie_id = L_f[greedy_index]
                    greedy_index += 1
                    if greedy_index >= len(L_f):
                        safety_counter = 100

                rating_count = sum([1 if x > 0 else 0 for x in X_obf[:, int(movie_id) - 1]])
                if rating_count > max_count[int(movie_id) - 1]:
                    continue
                if X_obf[index, int(movie_id) - 1] == 0:# and X_test [index, int(movie_id) - 1] ==0:
                    if rating_mode == 'higest':
                        X_obf[index, int(movie_id) - 1] = 5
                    elif rating_mode == 'avg':
                        X_obf[index, int(movie_id) - 1] = avg_ratings[int((movie_id) - 1) ] #int(index)
                    added += 1
                safety_counter += 1


    # row_sums = X.sum(axis=1)
    # np.save('test_X_obf.npy', X_obf)
    # X_obfu = X_obf * row_sums[:, np.newaxis]

    # X_obfu = Utils.denormalize(XX, X_obf)
    # X_obfu = scaler.inverse_transform(XX)

    # print(X_obfu)
    # output the data in a file:
    output_file = ""
    if dataset == 'ML':
        output_file = "ml1m/BlurMe/DoubleCount_BlurMe/" #"ml-1m/BlurMe/"
        with open(output_file + "TrainingSet_blurMe_ML1M_obfuscated_" + str(p) + "_" + sample_mode + "_" + rating_mode + "_top" + str(
                top) + ".dat", 'w') as f:
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(str(index_user + 1) + "::" + str(index_movie + 1) + "::" + str(
                            int(np.round(rating))) + "::000000000\n")

    elif dataset == 'Fx':
        output_file = "Flixster/DoubleCount_BlurMe/" # "Flixster/BlurMe/" FX/
        with open(output_file + "TrainingSet_FX_DCount_excludeTestSet_blurme_obfuscated_" + str(
                p) + "_" + sample_mode + "_" + rating_mode + "_top" + str(
                top) + ".dat", 'w') as f:
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(str(index_user + 1) + "::" + str(index_movie + 1) + "::" + str(
                            int(np.round(rating))) + "::000000000\n")
    elif dataset == 'LFM':
        output_file = "lastFM/BlurMe/DoubleCount_BlurMe/" # "Flixster/BlurMe/"
        with open(output_file + "TrainingSet_DCount_excludeTestSet_LFM_blurme_obfuscated_" + str(
                p) + "_" + sample_mode + "_" + rating_mode + "_top" + str(
                top) + ".dat", 'w') as f:
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(str(index_user + 1) + "::" + str(index_movie + 1) + "::" + str(
                            int(np.round(rating))) + "::000000000\n")

    else:
        with open("libimseti/LST_blurme_obfuscated_" + str(p) + "_" + sample_mode + "_" + rating_mode + "_top" + str(
                top) + ".dat", 'w') as f:
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(str(index_user+1) + "::" + str(index_movie+1) + "::" + str(
                            int(np.round(rating))) + "::000000000\n")

    return X_obf


def blurMe_100k():
    sample_mode = list(['random', 'sampled', 'greedy'])[2]
    rating_mode = list(['highest', 'avg', 'pred'])[1]

    # 1: get the set of most correlated movies, L_f and L_m:
    X = MD.load_user_item_matrix_100k()  # max_user=max_user, max_item=max_item)
    avg_ratings = np.zeros(shape=X.shape[1])
    for item_id in range(X.shape[1]):
        ratings = []
        for rating in X[:, item_id]:
            if rating > 0:
                ratings.append(rating)
        if len(ratings) == 0:
            avg_ratings[item_id] = 0
        else:
            avg_ratings[item_id] = np.average(ratings)

    T = MD.load_gender_vector_100k()  # max_user=max_user)
    X_train, T_train = X[0:int(0.8 * len(X))], T[0:int(0.8 * len(X))]
    X_test, T_test = X[int(0.8 * len(X)):], T[int(0.8 * len(X)):]

    from sklearn.model_selection import StratifiedKFold
    from sklearn.linear_model import LogisticRegression

    cv = StratifiedKFold(n_splits=10)
    coefs = []

    for train, test in cv.split(X_train, T_train):
        x, t = X_train[train], T_train[train]
        random_state = np.random.RandomState(0)
        model = LogisticRegression(penalty='l2', random_state=random_state)
        model.fit(x, t)
        coefs.append(model.coef_)

    coefs = np.average(coefs, axis=0)[0]
    coefs = [[coefs[i], i + 1] for i in range(len(coefs))]
    coefs = np.asarray(list(sorted(coefs)))
    L_m = coefs[:10, 1]
    L_f = coefs[coefs.shape[0] - 10:, 1]
    L_f = list(reversed(L_f))

    print(L_f)
    print("------")
    print(L_m)

    """
    movie_dict = MD.load_movie_id_dictionary_1m()
    print("males")
    for id in L_m:
        print(movie_dict[int(id)])

    print("females")
    for id in L_f:
        print(movie_dict[int(id)])
    """

    # Now, where we have the two lists, we can start obfuscating the data:
    X = MD.load_user_item_matrix_100k()
    X_obf = MD.load_user_item_matrix_100k()
    p = 0.1
    prob_m = [p / sum(L_m) for p in L_m]
    prob_f = [p / sum(L_f) for p in L_f]
    for index, user in enumerate(X):
        k = 0
        for rating in user:
            if rating > 0:
                k += 1
        k *= p
        greedy_index = 0
        #print(k)
        if T[index] == 1:
            added = 0
            safety_counter = 0
            while added < k and safety_counter < 100:
                # select a random movie:
                if sample_mode == 'random':
                    movie_id = L_m[np.random.randint(0, len(L_m))]
                elif sample_mode == 'sampled':
                    movie_id = L_m[np.random.choice(range(len(L_m)), p=prob_m)]
                elif sample_mode == 'greedy':
                    movie_id = L_m[greedy_index]
                    greedy_index += 1
                    if greedy_index >= len(L_m):
                        safety_counter = 100
                if X_obf[index, int(movie_id)-1] == 0:
                    if rating_mode == 'higest':
                        X_obf[index, int(movie_id) - 1] = 5
                    elif rating_mode == 'avg':
                        X_obf[index, int(movie_id) - 1] = avg_ratings[int(movie_id)]
                    added += 1
                safety_counter += 1
        elif T[index] == 0:
            added = 0
            safety_counter = 0
            while added < k and safety_counter < 100:
                # select a random movie:
                if sample_mode == 'random':
                    movie_id = L_f[np.random.randint(0, len(L_f))]
                elif sample_mode == 'sampled':
                    movie_id = L_f[np.random.choice(range(len(L_f)), p=prob_f)]
                elif sample_mode == 'greedy':
                    movie_id = L_f[greedy_index]
                    greedy_index += 1
                    if greedy_index >= len(L_f):
                        safety_counter = 100
                if X_obf[index, int(movie_id) - 1] == 0:
                    if rating_mode == 'higest':
                        X_obf[index, int(movie_id) - 1] = 5
                    elif rating_mode == 'avg':
                        X_obf[index, int(movie_id) - 1] = avg_ratings[int(movie_id)]
                    added += 1
                safety_counter += 1

    # output the data in a file:
    with open("ml-100k/blurme_obfuscated_" + str(p) + "_" + sample_mode + "_" + rating_mode + ".dat", 'w') as f:
        f.write("user_id,item_id,rating")
        for index_user, user in enumerate(X_obf):
            for index_movie, rating in enumerate(user):
                if rating > 0:
                    f.write(str(index_user + 1) + "," + str(index_movie + 1) + "," + str(int(rating)) + "\n")
    return X_obf


def rating_add_1m():
    # add a percentage of random ratings to a user:
    X = MD.load_user_item_matrix_1m()
    X_obf = MD.load_user_item_matrix_1m()
    percentage = 0.05
    for user_index, user in enumerate(X):
        nr_ratings = 0
        for rating in user:
            if rating > 0:
                nr_ratings += 1

        added = 0
        safety_counter = 0
        while added < nr_ratings*percentage and safety_counter < 100:
            index = np.random.randint(0,len(user))
            if X_obf[user_index, index] > 0:
                safety_counter += 1
                continue
            else:
                X_obf[user_index, index] = np.random.randint(1,6)

    # output the data in a file:
    with open("ml-1m/random_added_obfuscated_" + str(percentage) + ".dat", 'w') as f:
        for index_user, user in enumerate(X_obf):
            for index_movie, rating in enumerate(user):
                if rating > 0:
                    f.write(str(index_user + 1) + "::" + str(index_movie + 1) + "::" + str(
                        int(rating)) + "::000000000\n")
    return X_obf


def rating_swap_1m():
    plot = False
    low_bound, high_bound = 100, 1500
    # swap 0 ratings with non zero ratings:
    X = np.transpose(MD.load_user_item_matrix_1m())
    X_obf = np.transpose(MD.load_user_item_matrix_1m())
    nr_ratings = []
    for item in X:
        nr_rating = 0
        for rating in item:
            if rating > 0:
                nr_rating += 1
        nr_ratings.append(nr_rating)

    fig, (ax1, ax2) = plt.subplots(ncols=2, sharey=True)
    if plot:
        #plt.subplot(1,2,1)
        ax1.bar(range(1,len(X)+1), nr_ratings)
        ax1.set_xlabel("movie id")
        ax1.set_ylabel("nr ratings")

    # we want to remove ratings from movies that have more than 1500 ratings:
    amount_removed = 0
    for item_index, item in enumerate(X):
        if nr_ratings[item_index] > high_bound:
            indecies = np.argwhere(X[item_index,:] > 0)[:,0]
            indecies = np.random.choice(indecies, size=(nr_ratings[item_index]-high_bound,), replace=False)
            amount_removed += len(indecies)
            for i in indecies:
                X_obf[item_index, i] = 0
    """ To check if the removal is working
    
    nr_ratings = []
    for item in X_obf:
        nr_rating = 0
        for rating in item:
            if rating > 0:
                nr_rating += 1
        nr_ratings.append(nr_rating)
    if plot:
        plt.bar(range(1, len(X) + 1), nr_ratings)
        plt.xlabel("movie id")
        plt.ylabel("nr ratings")
        plt.show()
    
    """
    # now we want to add ratings to movies with a small number of ratings:
    print(np.asarray(nr_ratings))
    indecies = np.argwhere(np.asarray(nr_ratings) < low_bound)[:,0]
    print(indecies)
    nr_few_rated_movies = len(indecies)
    nr_to_be_added = amount_removed/nr_few_rated_movies
    print(nr_to_be_added)
    for item_index, item in enumerate(X):
        if nr_ratings[item_index] < low_bound:
            indecies = np.argwhere(X[item_index,:] == 0)[:,0]
            indecies = np.random.choice(indecies, size=(int(nr_to_be_added),), replace=False)
            for i in indecies:
                X_obf[item_index, i] = np.random.randint(1,6)

    """ To check if the removal and adding is working
    """
    nr_ratings = []
    for item in X_obf:
        nr_rating = 0
        for rating in item:
            if rating > 0:
                nr_rating += 1
        nr_ratings.append(nr_rating)
    if plot:
        #plt.subplot(1,2,2)
        ax2.bar(range(1, len(X) + 1), nr_ratings)
        ax2.set_xlabel("movie id")
        ax2.set_ylabel("nr ratings")
        plt.show()

    X_obf = np.transpose(X_obf)

    # output the data in a file:
    with open("ml-1m/rebalanced_(" + str(low_bound) + "," + str(high_bound) + ").dat", 'w') as f:
        for index_user, user in enumerate(X_obf):
            for index_movie, rating in enumerate(user):
                if rating > 0:
                    f.write(str(index_user + 1) + "::" + str(index_movie + 1) + "::" + str(
                        int(rating)) + "::000000000\n")

    return X_obf


def blurMePP():
    top = -1
    sample_mode = list(['random', 'sampled', 'greedy'])[2]
    removal_mode = list(['random', 'strategic'])[1]
    rating_mode = list(['avg', 'predicted'])[0]
    #id_index, index_id = MD.load_movie_id_index_dict()
    notice_factor = 2
    p = 0.05
    dataset = ['ML', 'Fx', 'LFM', 'Li'][2]
    if dataset == 'ML':
        X = MD.load_user_item_matrix_1m_all()
        # X = MD.load_user_item_matrix_1m_trainingSet()  # load_user_item_matrix_1m_trainingSet max_user=max_user, max_item=max_item)
        T = MD.load_gender_vector_1m()  # max_user=max_user)
        X_test = MD.load_user_item_matrix_1m_testSet()
        # X = MD.load_user_item_matrix_100k()
        # T = MD.load_gender_vector_100k()
    elif dataset == 'Fx':
        """import FlixsterData as FD
        #X, T, _ = FD.load_flixster_data_subset()
        X, T, _ = FD.load_flixster_data_subset_trainingSet()"""
        import FlixsterDataSub as FDS
        # X = FDS.load_user_item_matrix_FX_All()
        X = FDS.load_user_item_matrix_FX_TrainingSet()
        X_test = FDS.load_user_item_matrix_FX_Test()
        T = FDS.load_gender_vector_FX()
    elif dataset == 'LFM':
        import LastFMData as LFM
        # X = LFM.load_user_item_matrix_lfm_Train()  # LFM.load_user_item_matrix_lfm_All()
        X = LFM.load_user_item_matrix_lfm_All()  # load_user_item_matrix_lfm_Train LFM.load_user_item_matrix_lfm_All()
        T = LFM.load_gender_vector_lfm()
        X_test = LFM.load_user_item_matrix_lfm_Test()
    else:
        import LibimSeTiData as LD
        X, T, _ = LD.load_libimseti_data_subset()
    # X = Utils.normalizze(X)
    avg_ratings = np.zeros(shape=X.shape[1])
    initial_count = np.zeros(shape=X.shape[1])
    for item_id in range(X.shape[1]):
        ratings = []
        for rating in X[:, item_id]:
            if rating > 0:
                ratings.append(rating)
        if len(ratings) == 0:
            avg_ratings[item_id] = 0
        else:
            avg_ratings[item_id] = np.average(ratings)
        initial_count[item_id] = len(ratings)
    max_count = initial_count * notice_factor
    # 1: get the set of most correlated movies, L_f and L_m:
    """from sklearn.model_selection import StratifiedKFold
    from sklearn.linear_model import LogisticRegression

    cv = StratifiedKFold(n_splits=10)
    coefs = []
    avg_coefs = np.zeros(shape=(len(X[1]),))

    random_state = np.random.RandomState(0)
    for train, test in cv.split(X, T):
        x, t = X[train], T[train]
        model = LogisticRegression(penalty='l2', random_state=random_state)
        model.fit(x, t)
        # rank the coefs:
        ranks = ss.rankdata(model.coef_[0])
        coefs.append(ranks)
        # print(len(model.coef_[0]),len(x[0]))
        avg_coefs += model.coef_[0]

    coefs = np.average(coefs, axis=0)
    coefs = [[coefs[i], i + 1, avg_coefs[i]] for i in range(len(coefs))]
    coefs = np.asarray(list(sorted(coefs)))


    if top == -1:
        values = coefs[:,2]
        index_zero = np.where(np.abs(values) == np.min(np.abs(values)))
        top_male = index_zero[0][0]
        top_female = index_zero[0][-1]
        L_m = coefs[:top_male, 1][100:]
        # print(len(L_m))
        R_m = 2835 - coefs[:top_male, 0] #3952 2835
        C_m = np.abs(coefs[:top_male, 2])
        # C_m = [x for x in C_m if x > 2] # C_m[C_m <=  2]
        # print("C_m", type (C_m), "\n", C_m)
        L_f = coefs[coefs.shape[0] - top_female:, 1][100:]
        L_f = list(reversed(L_f))
        R_f = coefs[coefs.shape[0] - top_female:, 0]
        R_f = list(reversed(R_f))
        C_f = coefs[coefs.shape[0] - top_female:, 2]
        C_f = list(reversed(np.abs(C_f)))
        # C_f = [x for x in C_f if x > 2] # C_f[C_f <= 2]
        # print("C_f", type (C_f), "\n", C_f)

        # plt.plot(C_m, label = 'Male Coef', c= 'lightskyblue')
        # plt.plot(C_f, label = 'Female Coef', c= 'lightpink')
        # plt.axhline(y=2, color='crimson', linestyle='--')
        # plt.legend(loc="upper right")
        # plt.title("Male and Female coefficients on Flixster Data", fontsize=16, fontweight="bold")
        # plt.xlabel ('Features', fontsize=19)
        # plt.ylabel ('Coefficients', fontsize=19)
        # # plt.savefig("threshold_ML1M_IndicativeItems.pdf")
        # plt.show()

    else:
        L_m = coefs[:top, 1]
        R_m = 2835 -coefs[:top, 0] #3952 2835
        C_m = np.abs(coefs[:top, 2])
        L_f = coefs[coefs.shape[0]-top:, 1]
        L_f = list(reversed(L_f))
        R_f = coefs[coefs.shape[0]-top:, 0]
        R_f = list(reversed(R_f))
        C_f = coefs[coefs.shape[0]-top:, 2]
        C_f = list(reversed(np.abs(C_f)))

    # print(len(L_f))
    # Here we are trying to get all the less indicative items for F / M
    # Based on the plot we see that from 600 to the end the coefficients are <= 2
    L_ff = L_f.copy()
    print(L_ff)
    ## low indicative items
    # L_ff = L_ff [100:]
    # highly indicative items
    # L_ff = L_ff[:400]
    # print("L_ff:", L_ff, "\n\n", len(L_ff))
    L_ff = pd.DataFrame(L_ff)
    L_ff.to_csv('L_f_FX_Normalized.csv', index=False)
    # print("------")
    # print(len(L_m))
    L_mm = L_m.copy()
    print (L_mm)
    ## low indicative items
    # L_mm = L_mm [100:]
    # highly indicative items
    # L_mm = L_mm[:400]
    # print("L_mm:", L_mm, "\n\n", len( L_mm))
    L_mm = pd.DataFrame(L_mm)
    L_mm.to_csv('L_m_FX_Normalized.csv', index=False)"""
    # Now, where we have the two lists, we can start obfuscating the data:
    #X = MD.load_user_item_matrix_1m()
    #np.random.shuffle(X)
    #print(X.shape)

    X_obf = np.copy(X)
    total_added = 0
    for index, user in enumerate(X):
        print(index)
        k = 0
        for rating in user:
            if rating > 0:
                k += 1
        k *= p
        greedy_index_m = 0
        greedy_index_f = 0
        # print(k)
        added = 0
        if T[index] == 1:
            safety_counter = 0
            while added < k and safety_counter < 100:
                if greedy_index_m >= len(L_m):
                    safety_counter = 100
                    continue
                if sample_mode == 'greedy':
                    movie_id = L_m[greedy_index_m]
                if sample_mode == 'random':
                    movie_id = L_m[np.random.randint(0, len(L_m))]
                greedy_index_m += 1
                rating_count = sum([1 if x > 0 else 0 for x in X_obf[:, int(movie_id)-1]])
                if rating_count > max_count[int(movie_id)-1]:
                    continue
                if X_obf[index, int(movie_id) - 1] == 0:# and X_test [index, int(movie_id) - 1] ==0:
                    X_obf[index, int(movie_id) - 1] = avg_ratings[int(movie_id) - 1]
                    added += 1
                safety_counter += 1
        elif T[index] == 0:
            safety_counter = 0
            while added < k and safety_counter < 100:
                if greedy_index_f >= len(L_f):
                    safety_counter = 100
                    continue
                if sample_mode == 'greedy':
                    movie_id = L_f[greedy_index_f]
                if sample_mode == 'random':
                    movie_id = L_f[np.random.randint(0, len(L_f))]
                greedy_index_f += 1
                rating_count = sum([1 if x > 0 else 0 for x in X_obf[:, int(movie_id) - 1]])
                if rating_count > max_count[int(movie_id) - 1]:
                    continue

                if X_obf[index, int(movie_id) - 1] == 0:# and X_test [index, int(movie_id) - 1] ==0:
                    X_obf[index, int(movie_id) - 1] = avg_ratings[int(movie_id) - 1]
                    added += 1
                safety_counter += 1
        total_added += added

    # Now remove ratings from users that have more than 200 ratings equally:
    if removal_mode == "random":
        nr_many_ratings = 0
        for user in X:
            rating_count = sum([1 if x > 0 else 0 for x in user])
            if rating_count > 20: # 200 for ML1M and 300 for Flixster
                nr_many_ratings += 1
        nr_remove = total_added / nr_many_ratings

        for user_index, user in enumerate(X):
            rating_count = sum([1 if x > 0 else 0 for x in user])
            if rating_count > 20:
                to_be_removed_indecies = np.random.choice(np.argwhere(user > 0)[:, 0], size=(int(nr_remove),))#,replace=False)
                X_obf[user_index, to_be_removed_indecies] = 0
    else:
        nr_many_ratings = 0
        for user in X:
            rating_count = sum([1 if x > 0 else 0 for x in user])
            if rating_count > 20:
                nr_many_ratings += 1
        print("nr_many_ratings:", nr_many_ratings)
        print("total_added:", total_added)
        nr_remove = total_added / nr_many_ratings

        for user_index, user in enumerate(X):
            print("user: ", user_index)
            rating_count = sum([1 if x > 0 else 0 for x in user])
            if rating_count > 20:
                index_m = 0
                index_f = 0
                rem = 0
                if T[user_index] == 1:
                    safety_counter = 0
                    # We note that if we add safety_counter < 1000 in the while we have a higher accuracy than if we keep it in the if
                    while (rem < nr_remove) and safety_counter < 100:
                        if index_f >= len(L_f) :
                            safety_counter = 100
                            continue

                        if removal_mode == "random":
                            to_be_removed_indecies = np.random.choice(np.argwhere(user > 0)[:, 0],
                                                                      size=(int(nr_remove),),
                                                                      replace=False)  # , replace=False)
                        if removal_mode == "strategic":
                            to_be_removed_indecies = L_f[index_f]
                        index_f += 1

                        if X_obf[user_index, int(to_be_removed_indecies) - 1] != 0:
                            X_obf[user_index, int(to_be_removed_indecies) - 1] = 0
                            rem += 1
                        safety_counter += 1

                elif T[user_index] == 0:

                    while (rem < nr_remove) and safety_counter < 100:
                        if index_m >= len(L_m) :#and safety_counter < 1000:
                            safety_counter = 100
                            continue

                        if removal_mode == "random":
                            to_be_removed_indecies = np.random.choice(np.argwhere(user > 0)[:, 0],
                                                                      size=(int(nr_remove),),
                                                                      replace=False)  # , replace=False)
                        # X_obf[user_index, to_be_removed_indecies] = 0

                        if removal_mode == "strategic":
                            to_be_removed_indecies = L_m[index_m]
                        index_m += 1

                        if X_obf[user_index, int(to_be_removed_indecies) - 1] != 0:
                            X_obf[user_index, int(to_be_removed_indecies) - 1] = 0
                            rem += 1
                        safety_counter += 1

    
    # output the data in a file:
    output_file = ""
    if dataset == 'ML':
        output_file = "ml1m/"#"ml-1m/BlurMore/" ml-1m/BlurMore/Random_Removal/
        with open(output_file + "All_testSafe`Count_threshold20_ML1M_blurmepp_obfuscated_" + sample_mode + "_" +
                  str(p) + "_" + str(notice_factor) + "_" + str(removal_mode)  + ".dat",
                  'w') as f: # + "_" + str(removal_mode) + ".dat",
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(
                            str(index_user + 1) + "::" + str(index_movie + 1) + "::" + str(
                                int(np.round(rating))) + "::000000000\n")

    elif dataset == 'Fx':
        output_file = "Flixster/"#BlurMore/RandomRem/" # "Flixster/BlurMore/Greedy_Removal/" FX/
        with open(output_file + "All_testSafe`Count_threshold20_ExcludeTestSet_FX_blurmepp_obfuscated_" + sample_mode + "_" + str(p) + "_" + str(
                notice_factor) + "_" + str(removal_mode) + ".dat",
                  'w') as f:
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(str(index_user + 1) + "::" + str(index_movie + 1) + "::" + str(
                            int(np.round(rating))) + "::000000000\n")
    elif dataset == 'LFM':
        output_file = "lastFM/"#BlurMore/RandomRem/"
        with open(output_file + "All_testSafe`Count_LFM_blurmepp_ExcludeTestSet_obfuscated_" + sample_mode + "_" + str(p) + "_" +str( notice_factor) + "_" + str(removal_mode) + ".dat",
                      'w') as f:
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(str(index_user + 1) + "::" + str(index_movie + 1) + "::" + str(
                            int(np.round(rating))) + "::000000000\n")

    else:
        with open("libimseti/LST_blurmepp_obfuscated_" + sample_mode + "_" + str(p) + "_" + str(notice_factor) + ".dat", 'w') as f:
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(str(index_user+1) + "::" + str(index_movie+1) + "::" + str(
                            int(np.round(rating))) + "::000000000\n")


    return X_obf


#---------------------------------
"""Creation of Personalized list of indicative items """
def Personalized_list_User():
    notice_factor = 2
    item_choice = {}
    # lastFM/NN_LFM_TrainingSet_allUsers_KNN_fancy_imputation
    # lastFM/NN_LFM_All_allUsers_KNN_fancy_imputation
    with open('/Users/mslokom/Documents/RecSys_News/goodbook/With_Fancy_KNN/NN_GBD_TrainingSet_allUsers_KNN_fancy_imputation.json') as json_file:
        data = json.load(json_file)
    len_dict = {}
    for key, value in data.items():
        # print (value)
        length = []
        for v in value:
            # print (len(v))
            length.append(len(v))
        len_dict[int(key)] = length

    dataset = ['ML', 'Fx', 'LFM', 'Li'][0]
    if dataset == 'ML':  #
        # X = MD.load_user_item_matrix_1m_trainingSet()  #
        X = MD.load_user_item_matrix_1m_all()
        T = MD.load_gender_vector_1m()  #
    elif dataset == 'Fx':
        import FlixsterDataSub as FDS
        X = FDS.load_user_item_matrix_FX_All()
        # X = FDS.load_user_item_matrix_FX_TrainingSet()
        T = FDS.load_gender_vector_FX()
    elif dataset == 'LFM':
        import LastFMData as LFM
        X = LFM.load_user_item_matrix_lfm_All()
        # X = LFM.load_user_item_matrix_lfm_Train()
        T = LFM.load_gender_vector_lfm()
    else:
        import LibimSeTiData as LD
        X, T, _ = LD.load_libimseti_data_subset()

    avg_ratings = np.zeros(shape=X.shape[1])
    initial_count = np.zeros(shape=X.shape[1])
    for item_id in range(X.shape[1]):
        ratings = []
        for rating in X[:, item_id]:
            if rating > 0:
                ratings.append(rating)
        if len(ratings) == 0:
            avg_ratings[item_id] = 0
        else:
            avg_ratings[item_id] = np.average(ratings)
        initial_count[item_id] = len(ratings)
    max_count = initial_count * notice_factor
    # 1: get the set of most correlated movies, L_f and L_m:
    k = 100
    L_mm = list(map(int, L_m))
    L_mm = list(map(lambda x: x - 1, L_mm))[:k]
    L_ff = list(map(int, L_f))
    L_ff = list(map(lambda x: x - 1, L_ff))[:k]

    for z in range(len(X)):
        print(z)
        values = len_dict[z]
        lst_j = []
        # list of neighbors ordered / ranked by weight for user i
        user_item = list(np.argsort(values))  # [::-1])
        # lst = X_filled [z]
        # lst = list(map(lst.__getitem__, user_item))
        if (len(user_item) == len(values)):
            p = 0
            while p < len(values):
                if T[z] == 0:
                    f = user_item.pop(0)  # np.argmin (lst)
                    if f in L_ff:
                        if f not in lst_j:
                            lst_j.append(f)

                elif T[z] == 1:
                    m = user_item.pop(0)
                    if m in L_mm:
                        if m not in lst_j:
                            lst_j.append(m)
                p += 1
            item_choice[z] = lst_j
    print("item_choice: ", item_choice)
    # lastFM/With_Fancy_KNN/lfm_NN_TrainingSet_AllUsers_Neighbors_Weight_K_30_item_choice_Top500IndicativeItems.json
    with open("ml1m/With_Fancy_KNN/ML1M_NN_TrainingSet_AllUsers_Neighbors_Weight_K_30_item_choice_Top100IndicativeItems_noRemoval.json",
              "w") as fp:
        json.dump(item_choice, fp, cls=NpEncoder)
    
# --------------------------------

"""PerBlur without removal strategy function for obfuscating the user-item matrix"""


def PerBlur_No_Removal():

    sample_mode = list(['random', 'sampled', 'greedy'])[2]
    rating_mode = list(['highest', 'avg', 'pred'])[1]
    top = -1
    p = 0.05
    notice_factor = 2
    dataset = ['ML', 'Fx', 'LFM', 'Li'][0]
    if dataset == 'ML':  # load_user_item_matrix_1m_all
        X = MD.load_user_item_matrix_1m_all()  # load_user_item_matrix_1m_trainingSet
        # X = MD.load_user_item_matrix_1m_trainingSet()
        T = MD.load_gender_vector_1m()  #
        X_test = MD.load_user_item_matrix_1m_testSet()
        X_filled = MD.load_user_item_matrix_1m_complet()
    elif dataset == 'Fx':

        import FlixsterDataSub as FDS
        X = FDS.load_user_item_matrix_FX_All()
        # X = FDS.load_user_item_matrix_FX_TrainingSet()
        X_test = FDS.load_user_item_matrix_FX_Test()
        T = FDS.load_gender_vector_FX()
        X_filled = FDS.load_user_item_FX_Complet()
    elif dataset == 'LFM':
        import LastFMData as LFM
        # X = LFM.load_user_item_matrix_lfm_Train()  # LFM.load_user_item_matrix_lfm_All()
        X = LFM.load_user_item_matrix_lfm_All()  # load_user_item_matrix_lfm_Train LFM.load_user_item_matrix_lfm_All()
        T = LFM.load_gender_vector_lfm()
        X_test = LFM.load_user_item_matrix_lfm_Test()
        # X_filled = LFM.load_user_item_matrix_lfm_complet()
    else:
        import LibimSeTiData as LD
        X, T, _ = LD.load_libimseti_data_subset()


    avg_ratings = np.zeros(shape=X.shape[1])
    initial_count = np.zeros(shape=X.shape[1])
    for item_id in range(X.shape[1]):
        ratings = []
        for rating in X[:, item_id]:
            if rating > 0:
                ratings.append(rating)
        if len(ratings) == 0:
            avg_ratings[item_id] = 0
        else:
            avg_ratings[item_id] = np.average(ratings)
        initial_count[item_id] = len(ratings)
    max_count = initial_count * notice_factor
    # 1: get the set of most correlated movies, L_f and L_m:
    # lastFM/With_Fancy_KNN/lfm_NN_All_AllUsers_Neighbors_Weight_K_30_item_choice_TopAllIndicativeItems.json  lfm_NN_TrainingSet_AllUsers_Neighbors_Weight_K_30_item_choice_Top50IndicativeItems
    # ml1m/ML1M_NN_TrainingSet_AllUsers_Neighbors_Weight_K_30_item_choice_Top50IndicativeItems.json NN_All_AllUsers_Neighbors_Weight_K_30_item_choice_Top50IndicativeItems
    # Flixster/With_Fancy_KNN/FX_NN_TrainingSet_2370_AllUsers_Neighbors_Weight_K_30_item_choice_Top50IndicativeItems.json  FX_NN_All_2370_AllUsers_Neighbors_Weight_K_30_item_choice_Top50IndicativeItems
    # lfm_NN_TrainingSet_AllUsers_Neighbors_Weight_K_30_item_choice_Top1000IndicativeItems
    # ml-1m/PerBlur/test_Confidence_Score_Items_Selection/NN_All_AllUsers_Neighbors_Weight_K_30_item_choice_Top100IndicativeItems.json
    with open('ml-1m/PerBlur/test_Confidence_Score_Items_Selection/NN_All_AllUsers_Neighbors_Weight_K_30_item_choice_Top100IndicativeItems.json') as json_file:

        item_choice = json.load(json_file)

    # Now, where we have the two lists, we can start obfuscating the data:
    X_obf = np.copy(X)
    total_added = 0
    for index, user in enumerate(X):
        print(index)
        k = 0
        for rating in user:
            if rating > 0:
                k += 1
        k *= p
        greedy_index = 0
        added = 0
        mylist = list(item_choice.values())
        safety_counter = 0

        while added < k and safety_counter < 100: # 1000
            if greedy_index >= len(mylist[index]):
                safety_counter = 100
                continue
            if sample_mode == 'greedy':
                vec = mylist[index]
                movie_id = vec[greedy_index]
            if sample_mode == 'random':
                movie_id = vec[np.random.randint(0, len(vec))]
            greedy_index += 1
            rating_count = sum([1 if x > 0 else 0 for x in X_obf[:, movie_id]])
            if rating_count > max_count[movie_id]:
                continue

            if X_obf[index, movie_id] == 0:# and X_test [index, int(movie_id) ] == 0:
                X_obf[index, movie_id] = avg_ratings[int(movie_id)] # X_filled[index, movie_id] # avg_ratings[int(movie_id)] #
                added += 1
            safety_counter += 1
        total_added += added

    # output the data in a file:

    output_file = ""
    if dataset == 'ML':
        output_file = "ml1m/PerBlur/Top-100-NoRemoval/" # ml1m/
        with open(output_file + "All_thresh20_PerBlur_ML1M_obfuscated_Top100IndicativeItems_" + rating_mode + "_" + sample_mode + "_" + str(p) + "_" + str(notice_factor) + ".dat",
                  'w') as f:
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(
                            str(index_user + 1) + "::" + str(index_movie + 1) + "::" + str(
                                int(np.round(rating))) + "::000000000\n")


    elif dataset == 'Fx':
        output_file = "Flixster/BlurSome/Top-100-NoRemoval/"# "Flixster/BlurSome/Top-100/" FX
        with open(output_file + "All_thresh20_NoRemoval_blurSome_FX_obfuscated_Top100IndicativeItems_" + rating_mode + "_" + sample_mode + "_" + str(p) + "_" + str(
                notice_factor)  + ".dat",
                  'w') as f:
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(str(index_user + 1) + "::" + str(index_movie + 1) + "::" + str(
                            int(np.round(rating))) + "::000000000\n")
    elif dataset == 'LFM':
        output_file = "lastFM/PerBlur/Top-All-NoRemoval/"
        with open(output_file + "All_LFM_NoRemoval_PerBlur_obfuscated_TopAll_" + rating_mode + "_" + sample_mode + "_" + str(p) +
                  "_" +str( notice_factor)  + ".dat",
                      'w') as f:
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(str(index_user + 1) + "::" + str(index_movie + 1) + "::" + str(
                            int(np.round(rating))) + "::000000000\n")

    else:
        with open("libimseti/LST_blurmepp_obfuscated_" + sample_mode + "_" + str(p) + "_" + str(notice_factor) + ".dat", 'w') as f:
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(str(index_user+1) + "::" + str(index_movie+1) + "::" + str(
                            int(np.round(rating))) + "::000000000\n")


    return X_obf

        
        
#-----------------------------------------------

"""PerBlur with removal function for obfuscatibg the user-item matrix"""


def PerBlur():

    sample_mode = list(['random', 'sampled', 'greedy'])[2]
    rating_mode = list(['highest', 'avg', 'pred'])[1]
    removal_mode = list(['random', 'strategic'])[0]
    top = -1
    p = 0.05
    notice_factor = 2
    dataset = ['ML', 'Fx', 'LFM', 'Li'][2]
    if dataset == 'ML':  # load_user_item_matrix_1m_all
        X = MD.load_user_item_matrix_1m_trainingSet()  #
        T = MD.load_gender_vector_1m()  #
        X_test = MD.load_user_item_matrix_1m_testSet()
        X_filled = MD.load_user_item_matrix_1m_complet()
    elif dataset == 'Fx':

        import FlixsterDataSub as FDS
        # X = FDS.load_user_item_matrix_FX_All()
        X = FDS.load_user_item_matrix_FX_TrainingSet()
        X_test = FDS.load_user_item_matrix_FX_Test()
        T = FDS.load_gender_vector_FX()
        X_filled = FDS.load_user_item_FX_Complet()
    elif dataset == 'LFM':
        import LastFMData as LFM
        # X = LFM.load_user_item_matrix_lfm_Train()  # LFM.load_user_item_matrix_lfm_All()
        X = LFM.load_user_item_matrix_lfm_All()  # load_user_item_matrix_lfm_Train LFM.load_user_item_matrix_lfm_All()
        T = LFM.load_gender_vector_lfm()
        X_test = LFM.load_user_item_matrix_lfm_Test()
        X_filled = LFM.load_user_item_matrix_lfm_complet()
    else:
        import LibimSeTiData as LD
        X, T, _ = LD.load_libimseti_data_subset()


    avg_ratings = np.zeros(shape=X.shape[1])
    initial_count = np.zeros(shape=X.shape[1])
    for item_id in range(X.shape[1]):
        ratings = []
        for rating in X[:, item_id]:
            if rating > 0:
                ratings.append(rating)
        if len(ratings) == 0:
            avg_ratings[item_id] = 0
        else:
            avg_ratings[item_id] = np.average(ratings)
        initial_count[item_id] = len(ratings)
    max_count = initial_count * notice_factor
    # 1: get the set of most correlated movies, L_f and L_m:
    # lastFM/With_Fancy_KNN/lfm_NN_All_AllUsers_Neighbors_Weight_K_30_item_choice_TopAllIndicativeItems.json
    # ml1m/ML1M_NN_TrainingSet_AllUsers_Neighbors_Weight_K_30_item_choice_Top50IndicativeItems.json
    # Flixster/With_Fancy_KNN/FX_NN_TrainingSet_2370_AllUsers_Neighbors_Weight_K_30_item_choice_Top50IndicativeItems.json
    with open('lastFM/With_Fancy_KNN/lfm_NN_TrainingSet_AllUsers_Neighbors_Weight_K_30_item_choice_Top500IndicativeItems.json') as json_file:

        item_choice = json.load(json_file)

    # Now, where we have the two lists, we can start obfuscating the data:
    X_obf = np.copy(X)
    total_added = 0
    for index, user in enumerate(X):
        print(index)
        k = 0
        for rating in user:
            if rating > 0:
                k += 1
        k *= p
        greedy_index = 0
        added = 0
        mylist = list(item_choice.values())
        safety_counter = 0

        while added < k and safety_counter < 100:
            if greedy_index >= len(mylist[index]):
                safety_counter = 100
                continue
            if sample_mode == 'greedy':
                vec = mylist[index]
                movie_id = vec[greedy_index]
            if sample_mode == 'random':
                movie_id = vec[np.random.randint(0, len(vec))]
            greedy_index += 1
            rating_count = sum([1 if x > 0 else 0 for x in X_obf[:, movie_id]])
            if rating_count > max_count[movie_id]:
                continue

            if X_obf[index, movie_id] == 0:# and X_test [index, int(movie_id) ] == 0:
                X_obf[index, movie_id] = avg_ratings[int(movie_id)] # X_filled[index, movie_id] # avg_ratings[int(movie_id)] #
                added += 1
            safety_counter += 1
        total_added += added



    # Now remove ratings from users that have more than 200 ratings equally:
    if removal_mode == "strategic":
        nr_many_ratings = 0
        for user in X:
            rating_count = sum([1 if x > 0 else 0 for x in user])
            if rating_count > 20:
                nr_many_ratings += 1
        print("nbr user with profile length > 20: ", nr_many_ratings)
        print("total_added: ", total_added)
        nr_remove = total_added / nr_many_ratings
        print("nr_remove: ", nr_remove)

        for user_index, user in enumerate(X):
            print("user: ", user_index)
            rating_count = sum([1 if x > 0 else 0 for x in user])
            if rating_count > 20:
                index_m = 0
                index_f = 0
                rem = 0
                if T[user_index] == 1:
                    safety_counter = 0
                    # We note that if we add safety_counter < 1000 in the while we have a higher accuracy than if we keep it in the if
                    while (rem < nr_remove) and safety_counter < 100:
                        if index_f >= len(L_f):  # and safety_counter < 1000:
                            safety_counter = 100
                            continue

                        to_be_removed_indecies = L_f[index_f]
                        index_f += 1

                        if X_obf[user_index, int(to_be_removed_indecies)-1] != 0:
                            X_obf[user_index, int(to_be_removed_indecies)-1 ] = 0
                            rem += 1
                        safety_counter += 1

                elif T[user_index] == 0:

                    while (rem < nr_remove) and safety_counter < 100:
                        if index_m >= len(L_m):  # and safety_counter < 1000:
                            safety_counter = 100
                            continue

                        to_be_removed_indecies = L_m[index_m]
                        index_m += 1

                        if X_obf[user_index, int(to_be_removed_indecies) - 1] != 0:
                            X_obf[user_index, int(to_be_removed_indecies) - 1] = 0
                            rem += 1
                        safety_counter += 1
    else:
        # Now remove ratings from users that have more than 200 ratings equally:

        nr_many_ratings = 0
        for user in X:
            rating_count = sum([1 if x > 0 else 0 for x in user])
            if rating_count > 20:
                nr_many_ratings += 1
        nr_remove = total_added / nr_many_ratings

        for user_index, user in enumerate(X):
            rating_count = sum([1 if x > 0 else 0 for x in user])
            if rating_count > 20:
                to_be_removed_indecies = np.random.choice(np.argwhere(user > 0)[:, 0], size=(int(nr_remove),)) #  replace=False
                X_obf[user_index, to_be_removed_indecies] = 0


    # output the data in a file:

    output_file = ""
    if dataset == 'ML':
        output_file = "ml1m/PerBlur/Top-50/" # ml1m/
        with open(output_file + "TrainingSet_thresh20_PerBlur_ML1M_obfuscated_Top50IndicativeItems_" + rating_mode + "_" + sample_mode + "_" + str(p) + "_" + str(notice_factor) + "_" + str(removal_mode) + ".dat",
                  'w') as f:
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(
                            str(index_user + 1) + "::" + str(index_movie + 1) + "::" + str(
                                int(np.round(rating))) + "::000000000\n")


    elif dataset == 'Fx':
        output_file = "Flixster/BlurSome/Top-50-ExcludeTest/"# "Flixster/BlurSome/Top-100/" FX
        with open(output_file + "TrainingSet_thresh20_blurSome_FX_obfuscated_Top50IndicativeItems_" + rating_mode + "_" + sample_mode + "_" + str(p) + "_" + str(
                notice_factor) + "_" + str(removal_mode) + ".dat",
                  'w') as f:
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(str(index_user + 1) + "::" + str(index_movie + 1) + "::" + str(
                            int(np.round(rating))) + "::000000000\n")
    elif dataset == 'LFM':
        output_file = "lastFM/PerBlur/Top-500-RightSplit/"
        with open(output_file + "All_LFM_PerBlur_obfuscated_Top500_" + rating_mode + "_" + sample_mode + "_" + str(p) + "_" +str( notice_factor) + "_" + str(removal_mode) + ".dat",
                      'w') as f:
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(str(index_user + 1) + "::" + str(index_movie + 1) + "::" + str(
                            int(np.round(rating))) + "::000000000\n")

    else:
        with open("libimseti/LST_blurmepp_obfuscated_" + sample_mode + "_" + str(p) + "_" + str(notice_factor) + ".dat", 'w') as f:
            for index_user, user in enumerate(X_obf):
                for index_movie, rating in enumerate(user):
                    if rating > 0:
                        f.write(str(index_user+1) + "::" + str(index_movie+1) + "::" + str(
                            int(np.round(rating))) + "::000000000\n")


    return X_obf



# blurMe_100k()
# blurMe_1m()
blurMePP()
# Personalized_list_User() # This will create the lists of indicative items. It goes before the PerBlur function
# PerBlur ()
# PerBlur_No_Removal ()