From 825a4773768bae6ad5baf5c3514b1852152277cc Mon Sep 17 00:00:00 2001 From: Priyansh Kedia Date: Mon, 17 Jan 2022 13:34:17 +0530 Subject: [PATCH] added changes for user agents --- .../com/kedia/ogparser/JsoupNetworkCall.kt | 8 +--- .../com/kedia/ogparser/OpenGraphParser.kt | 31 +++++++-------- .../com/kedia/ogparser/UtilityFunctions.kt | 8 ++-- .../kedia/opengraphpreview/MainActivity.kt | 38 +++++++++++++++++-- 4 files changed, 53 insertions(+), 32 deletions(-) diff --git a/OGParser/src/main/java/com/kedia/ogparser/JsoupNetworkCall.kt b/OGParser/src/main/java/com/kedia/ogparser/JsoupNetworkCall.kt index 8d5ee65..3a715ca 100644 --- a/OGParser/src/main/java/com/kedia/ogparser/JsoupNetworkCall.kt +++ b/OGParser/src/main/java/com/kedia/ogparser/JsoupNetworkCall.kt @@ -1,5 +1,6 @@ package com.kedia.ogparser +import android.util.Log import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import org.jsoup.Jsoup @@ -7,7 +8,7 @@ import org.jsoup.Jsoup class JsoupNetworkCall { private val REFERRER = "http://www.google.com" - private val TIMEOUT = 10000 + private val TIMEOUT = 100000 private val DOC_SELECT_QUERY = "meta[property^=og:]" private val OPEN_GRAPH_KEY = "content" private val PROPERTY = "property" @@ -32,7 +33,6 @@ class JsoupNetworkCall { .execute() val doc = response.parse() - val ogTags = doc.select(DOC_SELECT_QUERY) when { ogTags.size > 0 -> @@ -64,10 +64,6 @@ class JsoupNetworkCall { } } catch (e: Exception) { e.printStackTrace() -// launch(Dispatchers.Main) { -// listener.onError(e.localizedMessage) -// } -// return@withContext null return null } diff --git a/OGParser/src/main/java/com/kedia/ogparser/OpenGraphParser.kt b/OGParser/src/main/java/com/kedia/ogparser/OpenGraphParser.kt index b64cd6a..5f3dc41 100644 --- a/OGParser/src/main/java/com/kedia/ogparser/OpenGraphParser.kt +++ b/OGParser/src/main/java/com/kedia/ogparser/OpenGraphParser.kt @@ -1,36 +1,30 @@ package com.kedia.ogparser import android.content.Context +import android.util.Log import kotlinx.coroutines.* -import org.jsoup.Jsoup import kotlin.coroutines.CoroutineContext class OpenGraphParser( private val listener: OpenGraphCallback, private var showNullOnEmpty: Boolean = false, - private val context: Context? = null + context: Context? = null ) { private val sharedPrefs: SharedPrefs? = context?.let { SharedPrefs(it) } private var url: String = "" - private val AGENTS = mutableListOf("Mozilla", "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)") - private val AGENT = "Mozilla" -// private val AGENT = "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)" - private val REFERRER = "http://www.google.com" - private val TIMEOUT = 10000 - private val DOC_SELECT_QUERY = "meta[property^=og:]" - private val OPEN_GRAPH_KEY = "content" - private val PROPERTY = "property" - private val OG_IMAGE = "og:image" - private val OG_DESCRIPTION = "og:description" - private val OG_URL = "og:url" - private val OG_TITLE = "og:title" - private val OG_SITE_NAME = "og:site_name" - private val OG_TYPE = "og:type" - + private val AGENTS = mutableListOf( + "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)", + "Mozilla", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36", + "WhatsApp/2.19.81 A", + "facebookexternalhit/1.1", + "facebookcatalog/1.0" + ) private val jsoupNetworkCall = JsoupNetworkCall() private var openGraphResult: OpenGraphResult? = null @@ -67,7 +61,8 @@ class OpenGraphParser( openGraphResult = jsoupNetworkCall.callUrl(url, it) val isResultNull = checkNullParserResult(openGraphResult) if (!isResultNull) { - return@forEach + openGraphResult?.let { sharedPrefs?.setOpenGraphResult(it, url) } + return@withContext openGraphResult } } diff --git a/OGParser/src/main/java/com/kedia/ogparser/UtilityFunctions.kt b/OGParser/src/main/java/com/kedia/ogparser/UtilityFunctions.kt index fa67bd8..8d665ff 100644 --- a/OGParser/src/main/java/com/kedia/ogparser/UtilityFunctions.kt +++ b/OGParser/src/main/java/com/kedia/ogparser/UtilityFunctions.kt @@ -1,10 +1,10 @@ package com.kedia.ogparser fun checkNullParserResult(openGraphResult: OpenGraphResult?): Boolean { - return (openGraphResult!!.title.isNullOrEmpty() || - openGraphResult!!.title.equals("null")) && - (openGraphResult!!.description.isNullOrEmpty() || - openGraphResult!!.description.equals( + return (openGraphResult?.title.isNullOrEmpty() || + openGraphResult?.title.equals("null")) && + (openGraphResult?.description.isNullOrEmpty() || + openGraphResult?.description.equals( "null" )) } \ No newline at end of file diff --git a/app/src/main/java/com/kedia/opengraphpreview/MainActivity.kt b/app/src/main/java/com/kedia/opengraphpreview/MainActivity.kt index 018b47d..dceb48a 100644 --- a/app/src/main/java/com/kedia/opengraphpreview/MainActivity.kt +++ b/app/src/main/java/com/kedia/opengraphpreview/MainActivity.kt @@ -1,21 +1,43 @@ package com.kedia.opengraphpreview -import androidx.appcompat.app.AppCompatActivity import android.os.Bundle -import android.os.Looper import android.util.Log +import androidx.appcompat.app.AppCompatActivity import com.kedia.ogparser.OpenGraphCallback import com.kedia.ogparser.OpenGraphParser import com.kedia.ogparser.OpenGraphResult import kotlinx.android.synthetic.main.activity_main.* class MainActivity : AppCompatActivity(), OpenGraphCallback { + + private val openGraphParser by lazy { OpenGraphParser(this, showNullOnEmpty = true) } + + private val LINKS_TO_TEST = mutableListOf( + "https://www.linkedin.com/posts/madhusmita-padhy_machinelearning-datascience-activity-6886390508722163712-yhQ0", + "https://www.youtube.com/watch?v=n3zsoX7bRlc", + "https://twitter.com/levelsio/status/1481942293108359168", + "https://stackoverflow.com/questions/44515769/conda-is-not-recognized-as-internal-or-external-command", + "https://github.com/Priyansh-Kedia/OpenGraphParser", + "https://chat.whatsapp.com/DdWAKRkt2VfAmd4OS47y7P", + "https://www.reddit.com/r/MachineLearning/comments/s3mjqf/deep_learning_interviews_hundreds_of_fully_solved/?utm_medium=android_app&utm_source=share", + "https://instagram.com/fcbarcelona?utm_medium=copy_link", + "https://www.facebook.com/groups/777946865955982/permalink/1385110621906267/", + "https://mobile.twitter.com/Twitter?ref_src=twsrc%5Egoogle%7Ctwcamp%5Eserp%7Ctwgr%5Eauthor" + ) + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContentView(R.layout.activity_main) - val openGraphParser = OpenGraphParser(this, showNullOnEmpty = true) - openGraphParser.parse("https://twitter.com/levelsio/status/1481942293108359168") + openGraphParser.parse(LINKS_TO_TEST.first()) + LINKS_TO_TEST.removeFirstOrNull() + +// for (link in LINKS_TO_TEST) { +// Handler().postDelayed({ +// openGraphParser.parse(link) +// }, 2000) +// Log.e("TAG!!!!", "called parse") +// } button.setOnClickListener { openGraphParser.parse(tview.text.toString()) @@ -25,10 +47,18 @@ class MainActivity : AppCompatActivity(), OpenGraphCallback { override fun onPostResponse(openGraphResult: OpenGraphResult) { Log.e("TAG!!!!", "response $openGraphResult") tview.setText(openGraphResult.toString()) + if (LINKS_TO_TEST.isNotEmpty()) { + openGraphParser.parse(LINKS_TO_TEST.first()) + LINKS_TO_TEST.removeFirstOrNull() + } } override fun onError(error: String) { Log.e("TAG!!!!", "$error") // tview.text = error + if (LINKS_TO_TEST.isNotEmpty()) { + openGraphParser.parse(LINKS_TO_TEST.first()) + LINKS_TO_TEST.removeFirstOrNull() + } } } \ No newline at end of file