-
Notifications
You must be signed in to change notification settings - Fork 6
/
Sentiment Analysis.json
1 lines (1 loc) · 7.24 KB
/
Sentiment Analysis.json
1
{"paragraphs":[{"title":"Importing functions","text":"%pyspark\n\nimport os\nfrom pyspark.sql.functions import explode,col,desc,asc, split","dateUpdated":"2018-05-02T05:12:14+0000","config":{"editorSetting":{"language":"python"},"colWidth":12,"editorMode":"ace/mode/python","title":true,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1525237412556_-2009634936","id":"20180425-232722_1986691420","dateCreated":"2018-05-02T05:03:32+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:3338"},{"title":"Reading 2 datasets for ratings and movies","text":"%pyspark\n\nbasic = spark.read.csv(\"/shared/users/avi/desktop/test/title.basics.tsv\",sep=\"\\t\")\nrating = spark.read.csv(\"/shared/users/avi/desktop/test/title.ratings.tsv\",sep=\"\\t\")","dateUpdated":"2018-05-02T05:03:32+0000","config":{"editorSetting":{"language":"python"},"colWidth":12,"editorMode":"ace/mode/python","title":true,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1525237412556_-2009634936","id":"20180425-232731_1618218922","dateCreated":"2018-05-02T05:03:32+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:3339"},{"title":"Filtering movies per year","text":"%pyspark\n\nfilterColumn = basic.select('_c0', '_c1', '_c2', '_c5','_c7','_c8').withColumnRenamed('_c0','tconst').withColumnRenamed('_c1', 'titletype').withColumnRenamed('_c2','title').withColumnRenamed('_c5','year').withColumnRenamed('_c7','runtime').withColumnRenamed('_c8','genre')\n\nfilterMovie = filterColumn.filter(\"titletype = 'movie'\").filter(\"year = '2017'\").select('tconst','title').orderBy(desc('year'))\n\nfilterRating = rating.select('_c0', '_c1', '_c2').withColumnRenamed('_c0','tconst').withColumnRenamed('_c1', 'avgRating').withColumnRenamed('_c2','votes')\n\noutput = filterMovie.join(filterRating, on=\"tconst\").filter(\"votes > 100000\").orderBy(desc('avgRating'),desc('votes'))\n","user":"anonymous","dateUpdated":"2018-05-02T05:19:20+0000","config":{"editorSetting":{"language":"python"},"colWidth":12,"editorMode":"ace/mode/python","title":true,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1525237412557_-2010019685","id":"20180425-232736_955310554","dateCreated":"2018-05-02T05:03:32+0000","dateStarted":"2018-05-02T05:19:20+0000","dateFinished":"2018-05-02T05:19:20+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:3340"},{"title":"Display the filtered dataset","text":"%pyspark\noutput.show()","user":"anonymous","dateUpdated":"2018-05-02T05:19:22+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"python"},"editorMode":"ace/mode/python","title":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+---------+--------------------+---------+------+\n| tconst| title|avgRating| votes|\n+---------+--------------------+---------+------+\n|tt2380307| Coco| 8.5|160979|\n|tt5027774|Three Billboards ...| 8.2|220136|\n|tt3315342| Logan| 8.1|472542|\n|tt1856101| Blade Runner 2049| 8.1|284099|\n|tt5013056| Dunkirk| 8.0|380432|\n|tt5726616|Call Me by Your Name| 8.0|100030|\n|tt3501632| Thor: Ragnarok| 7.9|301533|\n|tt5362988| Wind River| 7.8|119176|\n|tt1485796|The Greatest Showman| 7.8|109396|\n|tt3896198|Guardians of the ...| 7.7|364118|\n|tt5052448| Get Out| 7.7|296485|\n|tt3890160| Baby Driver| 7.7|291904|\n|tt0451279| Wonder Woman| 7.5|402624|\n|tt2250912|Spider-Man: Homec...| 7.5|306478|\n|tt1396484| It| 7.5|267463|\n|tt4425200|John Wick: Chapter 2| 7.5|224774|\n|tt5580390| The Shape of Water| 7.5|200002|\n|tt3450958|War for the Plane...| 7.5|166066|\n|tt4925292| Lady Bird| 7.5|114016|\n|tt2527336|Star Wars: The La...| 7.3|366323|\n+---------+--------------------+---------+------+\nonly showing top 20 rows\n\n"}]},"apps":[],"jobName":"paragraph_1525237910203_-1190396735","id":"20180502-051150_683362984","dateCreated":"2018-05-02T05:11:50+0000","dateStarted":"2018-05-02T05:19:22+0000","dateFinished":"2018-05-02T05:19:57+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:3341"},{"title":"Extracting Top 10 Movies","text":"%pyspark\nresult = output.select('title').limit(10)\nresult.show()","user":"anonymous","dateUpdated":"2018-05-02T05:15:38+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"python"},"editorMode":"ace/mode/python","title":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+--------------------+\n| title|\n+--------------------+\n| Coco|\n|Three Billboards ...|\n| Logan|\n| Blade Runner 2049|\n| Dunkirk|\n|Call Me by Your Name|\n| Thor: Ragnarok|\n|The Greatest Showman|\n| Wind River|\n|Guardians of the ...|\n+--------------------+\n\n"}]},"apps":[],"jobName":"paragraph_1525237471388_1747296867","id":"20180502-050431_338462954","dateCreated":"2018-05-02T05:04:31+0000","dateStarted":"2018-05-02T05:10:31+0000","dateFinished":"2018-05-02T05:11:01+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:3342"},{"text":"%pyspark\n\nfinalOutput.repartition(1).write.format(\"com.databricks.spark.csv\").option(\"header\", \"true\").save(\"/shared/users/avi/desktop/test/top10-2017\")","dateUpdated":"2018-05-02T05:18:06+0000","config":{"colWidth":12,"editorMode":"ace/mode/python","results":{},"enabled":true,"editorSetting":{"language":"python"}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1525237412559_-2009250187","id":"20180426-003719_100600887","dateCreated":"2018-05-02T05:03:32+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:3343"},{"text":"%pyspark\n","dateUpdated":"2018-05-02T05:03:32+0000","config":{"colWidth":12,"editorMode":"ace/mode/python","results":{},"enabled":true,"editorSetting":{"language":"python"}},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1525237412560_-1998861967","id":"20180502-005124_2147397326","dateCreated":"2018-05-02T05:03:32+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:3344"}],"name":"Sentiment Analysis","id":"2DD3RBY91","angularObjects":{"2DDFPQZZK:shared_process":[],"2DE6JACDG:shared_process":[],"2DDEMSS3V:shared_process":[],"2DCWRTS3Y:shared_process":[],"2DEA3Q68F:shared_process":[],"2DE8ZJAKA:shared_process":[],"2DD9PHCHX:shared_process":[],"2DCYE1F38:shared_process":[],"2DDBGEQKJ:shared_process":[],"2DBN5VQAZ:shared_process":[],"2DDTZ33HU:shared_process":[],"2DD9GFVB2:shared_process":[],"2DE8MRKTJ:shared_process":[],"2DDHMAEA4:shared_process":[],"2DEQF5WXC:shared_process":[],"2DEXXR7PE:shared_process":[],"2DBP1ZQR4:shared_process":[],"2DEYJBD3B:shared_process":[],"2DEYWKVZK:shared_process":[]},"config":{"looknfeel":"default","personalizedMode":"false"},"info":{}}