Merge pull request #333 from maarten-boot/master

update to latest whoisdomain
mboot-github · Sep 22, 2023 · 83112c5 · 83112c5
2 parents 659ed08 + 417add6
commit 83112c5
Show file tree

Hide file tree

Showing 28 changed files with 537 additions and 342 deletions.
diff --git a/makefile → Makefile b/makefile → Makefile
diff --git a/README.md b/README.md
@@ -48,83 +48,94 @@ Install the cli `whois` of your operating system if it is not present already
 
 Install `whois` package from your distribution (e.g apt install whois)
 
-```
-$pip install whois
 
->>> import whois
->>> domain = whois.query('google.com')
+        $pip install whois
 
->>> print(domain.__dict__)
-{
-	'expiration_date': datetime.datetime(2020, 9, 14, 0, 0),
-	'last_updated': datetime.datetime(2011, 7, 20, 0, 0),
-	'registrar': 'MARKMONITOR INC.',
-	'name': 'google.com',
-	'creation_date': datetime.datetime(1997, 9, 15, 0, 0)
-}
+        >>> import whois
+        >>> domain = whois.query('google.com')
 
->>> print(domain.name)
-google.com
+        >>> print(domain.__dict__)
+        {
+                'expiration_date': datetime.datetime(2020, 9, 14, 0, 0),
+                'last_updated': datetime.datetime(2011, 7, 20, 0, 0),
+                'registrar': 'MARKMONITOR INC.',
+                'name': 'google.com',
+                'creation_date': datetime.datetime(1997, 9, 15, 0, 0)
+        }
 
->>> print(domain.expiration_date)
-2020-09-14 00:00:00
-```
+        >>> print(domain.name)
+        google.com
+
+        >>> print(domain.expiration_date)
+        2020-09-14 00:00:00
 
 ## ccTLD & TLD support
 see the file: ./whois/tld_regexpr.py
 or call whois.validTlds()
 
 ## Issues
-Raise an issue https://github.com/DannyCork/python-whois/issues/new
+ * Raise an issue https://github.com/DannyCork/python-whois/issues/new
 
-## Changes:
-2022-06-09: maarten_boot:
+## Changes: 2022-06-09: maarten_boot:
  * the returned list of name_servers is now a sorted unique list and not a set
  * the help function whois.validTlds() now outputs the true tld with dots
 
-2022-09-27: maarten_boot
+## 2022-09-27: maarten_boot
  * add test2.py to replace test.py
  * ./test2.py -h will show the possible usage
  * all tests from the original program are now files in the ./tests directory
  * test can be done on all supported tld's with -a or --all and limitest by regex with -r <pattern> or --reg=<pattern>
 
-2022-11-04: maarten_boot
+## 2022-11-04: maarten_boot
  * add support for Iana example.com, example.net
 
-2022-11-07: maarten_boot
+## 2022-11-07: maarten_boot
  * add testing against static known data in dir: ./testdata/<domain>/output
  * test.sh will test all domains in testdata without actually calling whois, the input data is instead read from testdata/<domain>/input
 
-2022-11-11: maarten_boot
+## 2022-11-11: maarten_boot
+
  * add support for returning the raw data from the whois command: flag include_raw_whois_text
  * add support for handling unsupported domains via whois raw text only: flag return_raw_text_for_unsupported_tld
 
-2023-01-18: sorrowless
+## 2023-01-18: sorrowless
+
  * add an opportunity to specify maximum cache age
 
-2023-01-25: maarten_boot
+## 2023-01-25: maarten_boot
+
  * convert the tld file to a Dict, we now no longer need a mappper for python keywords or second level domains.
  * utf8 level domains also need no mapper anymore an can be added as is with a translation to xn--<something>
  * added xn-- tlds for all known utf-8 domains we currently have
  * we can now add new domains on the fly or change them:  whois.mergeExternalDictWithRegex(aDictToOverride) see example exampleExtend.py
 
-2023-01-27: maarten_boot
+## 2023-01-27: maarten_boot
+
  * add autodetect via iana tld file (this has only tld's)
  * add a central collection of all compiled regexes and reuse them: REG_COLLECTION_BY_KEY in _0_init_tld.py
  * refresh testdata now that tld has dot instead of _ if more then one level
  * add additional strings meaning domain does not exist
 
-2023-02-02: maarten_boot
+## 2023-02-02: maarten_boot
+
  * whois.QuotaStringsAdd(str) to add additional strings for over quota detection. whois.QuotaStrings() lists the current configured strings
  * whois.NoneStringsAdd(str) to add additional string for NoSuchDomainExists detection (whois.query() retuning None). whois.NoneStrings() lsts the current configured strings
  * suppress messages to stderr if not verbose=True
 
-2023-07-20: maarten_boot
-* sync with https://github.com/mboot-github/WhoisDomain; 1.20230720.1; (gov.tr), (com.ru, msk.ru, spb.ru), (option to preserve partial output after timeout)
-* sync with https://github.com/mboot-github/WhoisDomain; 1.20230720.2; add t_test hint support; fix some server hints
+## 2023-07-20: maarten_boot
+
+ * sync with https://github.com/mboot-github/WhoisDomain; 1.20230720.1; (gov.tr), (com.ru, msk.ru, spb.ru), (option to preserve partial output after timeout)
+ * sync with https://github.com/mboot-github/WhoisDomain; 1.20230720.2; add t_test hint support; fix some server hints
+
+## 2023-08-21: mboot-github (maarten_boot)
+
+ * abandon any python below 3.9 (mypy compatibilities)
+ * major refactor into more object based approch and parameterContext
+ * allow custom caching backends (e.g. redis, dbm, ...)
 
-2023-08-21: mboot-github (maarten_boot)
-* abandon any python below 3.9 (mypy compatibilities)
-* major refactor into more object based approcj and paramaterContext
-* allow custom caching backends (e.g. redis, dbm, ...)
+## 2023-09-22 see new paramaters in whois/context/parameterContext.oy
 
+ * Sync with latest whoisdomain
+ * Allow cleaning up the http(s) info in the status response.
+ * Allow correlation with tld (pip install tld) public_suffix.
+ * Allow display of what whois-servers were used until we reach the final item.
diff --git a/whois/__init__.py b/whois/__init__.py
@@ -1,5 +1,17 @@
+# pylint: disable=duplicate-code
+"""
+Module providing all public accessible functions and data for the whoisdomain package
+
+## optional modules supported:
+
+- if the tld library is installed you can use the `withPublicSuffix:bool` option
+
+All public data is vizible via the __all__ List
+"""
+
 import sys
 import os
+import logging
 
 from functools import wraps
 
@@ -56,12 +68,15 @@
     initLastWhois,
 )
 
+log = logging.getLogger(__name__)
+logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
+
 HAS_REDIS = False
 try:
     import redis
 
     HAS_REDIS = True
-except Exception as e:
+except ImportError as e:
     _ = e
 
 if HAS_REDIS:
@@ -82,7 +97,7 @@
     import tld as libTld
 
     TLD_LIB_PRESENT = True
-except Exception as e:
+except ImportError as e:
     _ = e  # ignore any error
 
 __all__ = [
@@ -196,6 +211,8 @@ def query(
     tryInstallMissingWhoisOnWindows: bool = False,
     shortResponseLen: int = 5,
     withPublicSuffix: bool = False,
+    extractServers: bool = False,
+    stripHttpStatus: bool = False,
     # if you use pc as argument all above params (except domain are ignored)
 ) -> Optional[Domain]:
     # see documentation about paramaters in parameterContext.py
@@ -223,10 +240,12 @@ def query(
             withPublicSuffix=withPublicSuffix,
             shortResponseLen=shortResponseLen,
             tryInstallMissingWhoisOnWindows=tryInstallMissingWhoisOnWindows,
+            extractServers=extractServers,
+            stripHttpStatus=stripHttpStatus,
         )
 
-    if verbose:
-        print(pc, file=sys.stderr)
+    msg = f"{pc}"
+    log.debug(msg)
 
     return q2(domain=domain, pc=pc)
 

diff --git a/whois/cache/dbmCache.py b/whois/cache/dbmCache.py
@@ -1,10 +1,15 @@
-import sys
+# import sys
 import dbm
+import os
+import logging
 
 from typing import (
     Optional,
 )
 
+log = logging.getLogger(__name__)
+logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
+
 
 class DBMCache:
     def __init__(
@@ -14,22 +19,20 @@ def __init__(
     ) -> None:
         self.verbose = verbose
         self.dbmFile = dbmFile
-        if self.verbose:
-            print(f"{type(self).__name__} verbose: {self.verbose}", file=sys.stderr)
 
     def get(
         self,
         keyString: str,
     ) -> Optional[str]:
-        if self.verbose:
-            print(f"{type(self).__name__} get: {keyString}", file=sys.stderr)
+        msg = f"{type(self).__name__} get: {keyString}"
+        log.debug(msg)
 
         with dbm.open(self.dbmFile, "c") as db:
             data = db.get(keyString, None)
             if data:
                 sdata: str = data.decode("utf-8")
-                if self.verbose:
-                    print(sdata, file=sys.stderr)
+                msg = f"{sdata}"
+                log.debug(msg)
                 return sdata
         return None
 
@@ -38,8 +41,8 @@ def put(
         keyString: str,
         data: str,
     ) -> str:
-        if self.verbose:
-            print(f"{type(self).__name__} put: {keyString}", file=sys.stderr)
+        msg = f"{type(self).__name__} put: {keyString}"
+        log.debug(msg)
 
         with dbm.open(self.dbmFile, "c") as db:
             db[keyString] = bytes(data, "utf-8")

diff --git a/whois/cache/dummyCache.py b/whois/cache/dummyCache.py
@@ -1,32 +1,31 @@
-import sys
+# import sys
+import os
+import logging
 
 from typing import (
     Optional,
 )
 
+log = logging.getLogger(__name__)
+logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
+
 
 class DummyCache:
     def __init__(
         self,
         verbose: bool = False,
     ) -> None:
         self.verbose = verbose
-        if self.verbose:
-            print(f"{type(self).__name__} verbose: {self.verbose}", file=sys.stderr)
 
     def get(
         self,
         keyString: str,
     ) -> Optional[str]:
-        if self.verbose:
-            print(f"{type(self).__name__} get: {keyString}", file=sys.stderr)
         return None
 
     def put(
         self,
         keyString: str,
         data: str,
     ) -> str:
-        if self.verbose:
-            print(f"{type(self).__name__} put: {keyString}", file=sys.stderr)
         return data
diff --git a/whois/cache/redisCache.py b/whois/cache/redisCache.py
@@ -1,16 +1,23 @@
 #! /usr/bin/env python3
+# pylint: disable=duplicate-code
+# pylint disable=broad-exception-caught
+
+import os
+import logging
 
-import sys
 from typing import (
     Optional,
 )
 
+log = logging.getLogger(__name__)
+logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
+
 HAS_REDIS = False
 try:
     import redis
 
     HAS_REDIS = True
-except Exception as e:
+except ImportError as e:
     _ = e
 
 if HAS_REDIS:
@@ -28,21 +35,14 @@ def __init__(self, verbose: bool = False, host: str = "localhost", port: int = 6
                 connection_pool=self.pool,
             )
 
-            if self.verbose:
-                print(f"{type(self).__name__} verbose: {self.verbose}", file=sys.stderr)
-
         def get(
             self,
             keyString: str,
         ) -> Optional[str]:
-            if self.verbose:
-                print(f"{type(self).__name__} get: {keyString}", file=sys.stderr)
 
             data = self.redis.get(keyString)
             if data:
                 sdata: str = data.decode("utf-8")
-                if self.verbose:
-                    print(sdata, file=sys.stderr)
                 return sdata
             return None
 
@@ -51,8 +51,6 @@ def put(
             keyString: str,
             data: str,
         ) -> str:
-            if self.verbose:
-                print(f"{type(self).__name__} put: {keyString}", file=sys.stderr)
 
             self.redis.set(
                 keyString,