Merge pull request #1160 from yyyangw/feature/gunicorn_update

增加gunicorn架构上报指标配置
Tencent · Aug 19, 2024 · 49c4435 · 49c4435
2 parents 2fbad7e + 7ca1792
commit 49c4435
Show file tree

Hide file tree

Showing 10 changed files with 231 additions and 22 deletions.
diff --git a/helm-charts/tca/README.md b/helm-charts/tca/README.md
@@ -206,12 +206,13 @@ Helm will deploy `mariadb` by default. Reference: [bitnami/mariadb](https://gith
 | `tca.web.logPath`                            | TCA Web nginx log path                                                   | `/var/log/nginx`          |
 
 ### TCA Main configuration parameters
-| Name                                         | Description                                                              | Value                     |
-| -------------------------------------------- | ------------------------------------------------------------------------ | ------------------------- |
-| `tca.main.image.repository`                  | TCA Main image repository                                                | `tencenttca/tca-main`     |
-| `tca.main.image.tag`                         | TCA Main image tag                                                       | `latest`                  |
-| `tca.main.server.processnum`                 | TCA Main server process num                                              | `8`                       |
-| `tca.main.worker.num`                        | TCA Main worker num for async starting analysis and handle periodic task | `2`                       |
+| Name                                         | Description                                                              | Value                 |
+| -------------------------------------------- | ------------------------------------------------------------------------ |-----------------------|
+| `tca.main.image.repository`                  | TCA Main image repository                                                | `tencenttca/tca-main` |
+| `tca.main.image.tag`                         | TCA Main image tag                                                       | `latest`              |
+| `tca.main.server.processnum`                 | TCA Main server process num                                              | `8`                   |
+| `tca.main.worker.num`                        | TCA Main worker num for async starting analysis and handle periodic task | `2`                   |
+| `tca.main.server.multiprocDir`               | TCA Main server storing monitoring indicator data path                   | `multiproc-tmp`       |
 
 #### TCA Main setting configuration
 | Name                                         | Description                                                              | Value                     |
@@ -231,12 +232,14 @@ Helm will deploy `mariadb` by default. Reference: [bitnami/mariadb](https://gith
 
 
 ### TCA Analysis Configuration parameters
-| Name                                         | Description                                                              | Value                     |
-| -------------------------------------------- | ------------------------------------------------------------------------ | ------------------------- |
-| `tca.analysis.image.repository`              | TCA Analysis image repository                                            | `tencenttca/tca-analysis` |
-| `tca.analysis.image.tag`                     | TCA Analysis image tag                                                   | `latest`                  |
-| `tca.analysis.server.processnum`             | TCA Analysis server process num                                          | `8`                       |
-| `tca.analysis.worker.num`                    | TCA Analysis worker num for async saving lint and metric data            | `2`                       |
+| Name                                         | Description                                                   | Value                     |
+|----------------------------------------------|---------------------------------------------------------------|---------------------------|
+| `tca.analysis.image.repository`              | TCA Analysis image repository                                 | `tencenttca/tca-analysis` |
+| `tca.analysis.image.tag`                     | TCA Analysis image tag                                        | `latest`                  |
+| `tca.analysis.server.processnum`             | TCA Analysis server process num                               | `8`                       |
+| `tca.analysis.worker.num`                    | TCA Analysis worker num for async saving lint and metric data | `2`                       |
+| `tca.analysis.server.multiprocDir`           | TCA Analysis server storing monitoring indicator data path    | `multiproc-tmp`           |
+
 
 #### TCA Analysis setting configuration
 | Name                                         | Description                                                              | Value                     |
@@ -254,11 +257,12 @@ Helm will deploy `mariadb` by default. Reference: [bitnami/mariadb](https://gith
 
 
 ### TCA Login Configuration parameters
-| Name                                         | Description                                                              | Value                     |
-| -------------------------------------------- | ------------------------------------------------------------------------ | ------------------------- |
-| `tca.login.image.repository`                 | TCA Login image repository                                               | `tencenttca/tca-login`    |
-| `tca.login.image.tag`                        | TCA Login image tag                                                      | `latest`                  |
-| `tca.login.server.processnum`                | TCA Login server process num                                             | `8`                       |
+| Name                             | Description                                              | Value                    |
+|----------------------------------|----------------------------------------------------------|--------------------------|
+| `tca.login.image.repository`     | TCA Login image repository                               | `tencenttca/tca-login`   |
+| `tca.login.image.tag`            | TCA Login image tag                                      | `latest`                 |
+| `tca.login.server.processnum`    | TCA Login server process num                             | `8`                      |
+| `tca.login.server.multiprocDir`  | TCA Login server storing monitoring indicator data path  | `multiproc-tmp`          |
 
 
 #### TCA Login setting configuration
@@ -271,10 +275,11 @@ Helm will deploy `mariadb` by default. Reference: [bitnami/mariadb](https://gith
 | `tca.login.settings.customDB.user`           | TCA Login custom mysql server user                                       | `""`                      |
 
 ### TCA File Configuration parameters
-| Name                                         | Description                                                              | Value                     |
-| -------------------------------------------- | ------------------------------------------------------------------------ | ------------------------- |
-| `tca.file.image.repository`                  | TCA file image repository                                                | `tencenttca/tca-file`     |
-| `tca.file.image.tag`                         | TCA file image tag                                                       | `latest`                  |
+| Name                            | Description                                             | Value                 |
+|---------------------------------|---------------------------------------------------------|-----------------------|
+| `tca.file.image.repository`     | TCA file image repository                               | `tencenttca/tca-file` |
+| `tca.file.image.tag`            | TCA file image tag                                      | `latest`              |
+| `tca.file.server.multiprocDir`  | TCA file server storing monitoring indicator data path  | `multiproc-tmp`       |
 
 #### TCA File setting configuration
 | Name                                         | Description                                                              | Value                     |

diff --git a/helm-charts/tca/templates/analysis/configmap.yaml b/helm-charts/tca/templates/analysis/configmap.yaml
@@ -18,6 +18,7 @@ data:
   ANALYSIS_DEBUG_MODE: {{ .Values.tca.analysis.settings.base.debugMode | quote }}
   ANALYSIS_DB_NAME: {{ .Values.tca.analysis.settings.dbName }}
   ANALYSIS_REDIS_DBID: {{ .Values.tca.analysis.settings.redisDBId | quote }}
+  MY_PROMETHEUS_MULTIPROC_DIR: {{ .Values.tca.analysis.server.multiprocDir }}
   # DB相关配置，优先使用 Values.tca.analysis.settings.customDB 配置，默认使用 tca.database
   ANALYSIS_DB_USER: {{ .Values.tca.analysis.settings.customDB.user | default ( include "tca.database.username" . ) }}
   ANALYSIS_DB_PASSWORD: {{ .Values.tca.analysis.settings.customDB.password | default ( include "tca.database.password" . ) }}

diff --git a/helm-charts/tca/templates/file/configmap-env.yaml b/helm-charts/tca/templates/file/configmap-env.yaml
@@ -28,6 +28,7 @@ data:
   FILE_MINIO_ACCESS_KEY: {{ .Values.tca.file.settings.minio.accessKey | quote }}
   FILE_MINIO_SECRET_KEY: {{ .Values.tca.file.settings.minio.secretKey | quote }}
   FILE_DB_NAME: {{ .Values.tca.file.settings.dbName | quote }}
+  MY_PROMETHEUS_MULTIPROC_DIR: {{ .Values.tca.file.server.multiprocDir }}
   # DB相关配置，优先使用 Values.tca.file.settings.customDB 配置，默认使用 tca.database
   FILE_DB_USER: {{ .Values.tca.file.settings.customDB.user | default ( include "tca.database.username" . ) }}
   FILE_DB_PASSWORD: {{ .Values.tca.file.settings.customDB.password | default ( include "tca.database.password" . ) }}

diff --git a/helm-charts/tca/templates/login/configmap.yaml b/helm-charts/tca/templates/login/configmap.yaml
@@ -18,6 +18,7 @@ data:
   DJANGO_SETTINGS_MODULE: {{ .Values.tca.login.settings.base.module }}
   LOGIN_DEBUG_MODE: {{ .Values.tca.login.settings.base.debugMode | quote }}
   LOGIN_DB_NAME: {{ .Values.tca.login.settings.dbName }}
+  MY_PROMETHEUS_MULTIPROC_DIR: {{ .Values.tca.login.server.multiprocDir }}
   # DB相关配置，优先使用 Values.tca.login.settings.customDB 配置，默认使用 tca.database
   LOGIN_DB_USER: {{ .Values.tca.login.settings.customDB.user | default ( include "tca.database.username" . ) }}
   LOGIN_DB_PASSWORD: {{ .Values.tca.login.settings.customDB.password | default ( include "tca.database.password" . ) }}

diff --git a/helm-charts/tca/templates/main/configmap.yaml b/helm-charts/tca/templates/main/configmap.yaml
@@ -23,6 +23,7 @@ data:
   HTTPS_CLONE_FLAG: {{ .Values.tca.main.settings.base.httpsCloneFlag | quote }}
   MAIN_DB_NAME: {{ .Values.tca.main.settings.dbName }}
   MAIN_REDIS_DBID: {{ .Values.tca.main.settings.redisDBId | quote }}
+  MY_PROMETHEUS_MULTIPROC_DIR: {{ .Values.tca.main.server.multiprocDir }}
   # DB相关配置，优先使用 Values.tca.main.settings.customDB 配置，默认使用 tca.database
   MAIN_DB_USER: {{ .Values.tca.main.settings.customDB.user | default ( include "tca.database.username" . ) }}
   MAIN_DB_PASSWORD: {{ .Values.tca.main.settings.customDB.password | default ( include "tca.database.password" . ) }}

diff --git a/helm-charts/tca/values.yaml b/helm-charts/tca/values.yaml
@@ -221,10 +221,12 @@ tca:
     ## 
     server:
       ## @param tca.main.server.port A port for main server listening
-      ## @param tca.analysis.server.processNum The number of processes main server is running
+      ## @param tca.main.server.processNum The number of processes main server is running
+      ## @param tca.main.server.multiprocDir Path for storing monitoring indicator data
       ##
       port: 80
       processNum: 8
+      multiprocDir: "multiproc-tmp"
       resources:
         limits:
           cpu: "1000m"
@@ -333,9 +335,11 @@ tca:
     server:
       ## @param tca.analysis.server.port A port for analysis server listening
       ## @param tca.analysis.server.processNum The number of processes analysis server is running
+      ## @param tca.analysis.server.multiprocDir Path for storing monitoring indicator data
       ##
       port: 80
       processNum: 8
+      multiprocDir: "multiproc-tmp"
       resources:
         limits:
           cpu: "1000m"
@@ -430,9 +434,11 @@ tca:
     server:
       ## @param tca.login.server.port A port for login server listening
       ## @param tca.login.server.processNum The number of processes login server is running
+      ## @param tca.login.server.multiprocDir Path for storing monitoring indicator data
       ##
       port: 80
       processNum: 8
+      multiprocDir: "multiproc-tmp"
       resources:
         limits:
           cpu: "1000m"
@@ -493,9 +499,11 @@ tca:
     server:
       ## @param tca.file.server.port A port for file server listening
       ## @param tca.file.server.processNum The number of processes file server is running
+      ## @param tca.file.server.multiprocDir Path for storing monitoring indicator data
       ##
       port: 8804
       processNum: 8
+      multiprocDir: "multiproc-tmp"
       resources:
         limits: 
           cpu: "1000m"

diff --git a/server/projects/analysis/analysis.gunicorn.conf.py b/server/projects/analysis/analysis.gunicorn.conf.py
@@ -9,6 +9,54 @@
 """
 import os
 
+
+def on_starting(server):
+    if os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR', None):
+        os.environ['PROMETHEUS_MULTIPROC_DIR'] = os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR')
+        BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        multiproc_dir = os.environ.get('PROMETHEUS_MULTIPROC_DIR')
+
+        try:
+            if not os.path.exists(multiproc_dir):
+                os.makedirs(multiproc_dir)
+        except Exception:
+            os.environ['PROMETHEUS_MULTIPROC_DIR'] = BASE_DIR
+
+
+def post_fork(server, worker):
+    if os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR', None):
+        try:
+            from prometheus_client import values
+            values.ValueClass = values.MultiProcessValue(os.getpid)
+        except ImportError:
+            pass
+
+
+def child_exit(server, worker):
+    if os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR', None):
+        mark_all_process_dead(worker.pid)
+
+
+def mark_all_process_dead(pid, path=None):
+    if path is None:
+        path = os.environ.get('PROMETHEUS_MULTIPROC_DIR')
+
+    try:
+        from prometheus_client import Gauge
+        import glob
+        for mode in Gauge._MULTIPROC_MODES:
+            for f in glob.glob(os.path.join(path, f'gauge_{mode}_{pid}.db')):
+                os.remove(f)
+    except ImportError:
+        pass
+
+    try:
+        os.remove(os.path.join(path, f'counter_{pid}.db'))
+        os.remove(os.path.join(path, f'histogram_{pid}.db'))
+    except Exception:
+        pass
+
+
 project_path = os.path.dirname(os.path.abspath(__file__))
 host = os.environ.get("ANALYSIS_SERVER_HOST", "0.0.0.0")
 port = os.environ.get("ANALYSIS_SERVER_PORT", 8002)

diff --git a/server/projects/file/file.gunicorn.conf.py b/server/projects/file/file.gunicorn.conf.py
@@ -9,6 +9,54 @@
 """
 import os
 
+
+def on_starting(server):
+    if os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR', None):
+        os.environ['PROMETHEUS_MULTIPROC_DIR'] = os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR')
+        BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        multiproc_dir = os.environ.get('PROMETHEUS_MULTIPROC_DIR')
+
+        try:
+            if not os.path.exists(multiproc_dir):
+                os.makedirs(multiproc_dir)
+        except Exception:
+            os.environ['PROMETHEUS_MULTIPROC_DIR'] = BASE_DIR
+
+
+def post_fork(server, worker):
+    if os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR', None):
+        try:
+            from prometheus_client import values
+            values.ValueClass = values.MultiProcessValue(os.getpid)
+        except ImportError:
+            pass
+
+
+def child_exit(server, worker):
+    if os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR', None):
+        mark_all_process_dead(worker.pid)
+
+
+def mark_all_process_dead(pid, path=None):
+    if path is None:
+        path = os.environ.get('PROMETHEUS_MULTIPROC_DIR')
+
+    try:
+        from prometheus_client import Gauge
+        import glob
+        for mode in Gauge._MULTIPROC_MODES:
+            for f in glob.glob(os.path.join(path, f'gauge_{mode}_{pid}.db')):
+                os.remove(f)
+    except ImportError:
+        pass
+
+    try:
+        os.remove(os.path.join(path, f'counter_{pid}.db'))
+        os.remove(os.path.join(path, f'histogram_{pid}.db'))
+    except Exception:
+        pass
+
+
 project_path = os.path.dirname(os.path.abspath(__file__))
 host = os.environ.get("FILE_SERVER_HOST", "0.0.0.0")
 port = os.environ.get("FILE_SERVER_PORT", 8804)

diff --git a/server/projects/login/login.gunicorn.conf.py b/server/projects/login/login.gunicorn.conf.py
@@ -9,6 +9,54 @@
 """
 import os
 
+
+def on_starting(server):
+    if os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR', None):
+        os.environ['PROMETHEUS_MULTIPROC_DIR'] = os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR')
+        BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        multiproc_dir = os.environ.get('PROMETHEUS_MULTIPROC_DIR')
+
+        try:
+            if not os.path.exists(multiproc_dir):
+                os.makedirs(multiproc_dir)
+        except Exception:
+            os.environ['PROMETHEUS_MULTIPROC_DIR'] = BASE_DIR
+
+
+def post_fork(server, worker):
+    if os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR', None):
+        try:
+            from prometheus_client import values
+            values.ValueClass = values.MultiProcessValue(os.getpid)
+        except ImportError:
+            pass
+
+
+def child_exit(server, worker):
+    if os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR', None):
+        mark_all_process_dead(worker.pid)
+
+
+def mark_all_process_dead(pid, path=None):
+    if path is None:
+        path = os.environ.get('PROMETHEUS_MULTIPROC_DIR')
+
+    try:
+        from prometheus_client import Gauge
+        import glob
+        for mode in Gauge._MULTIPROC_MODES:
+            for f in glob.glob(os.path.join(path, f'gauge_{mode}_{pid}.db')):
+                os.remove(f)
+    except ImportError:
+        pass
+
+    try:
+        os.remove(os.path.join(path, f'counter_{pid}.db'))
+        os.remove(os.path.join(path, f'histogram_{pid}.db'))
+    except Exception:
+        pass
+
+
 project_path = os.path.dirname(os.path.abspath(__file__))
 host = os.environ.get("LOGIN_SERVER_HOST", "0.0.0.0")
 port = os.environ.get("LOGIN_SERVER_PORT", 8003)

diff --git a/server/projects/main/main.gunicorn.conf.py b/server/projects/main/main.gunicorn.conf.py
@@ -9,6 +9,54 @@
 """
 import os
 
+
+def on_starting(server):
+    if os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR', None):
+        os.environ['PROMETHEUS_MULTIPROC_DIR'] = os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR')
+        BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        multiproc_dir = os.environ.get('PROMETHEUS_MULTIPROC_DIR')
+
+        try:
+            if not os.path.exists(multiproc_dir):
+                os.makedirs(multiproc_dir)
+        except Exception:
+            os.environ['PROMETHEUS_MULTIPROC_DIR'] = BASE_DIR
+
+
+def post_fork(server, worker):
+    if os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR', None):
+        try:
+            from prometheus_client import values
+            values.ValueClass = values.MultiProcessValue(os.getpid)
+        except ImportError:
+            pass
+
+
+def child_exit(server, worker):
+    if os.environ.get('MY_PROMETHEUS_MULTIPROC_DIR', None):
+        mark_all_process_dead(worker.pid)
+
+
+def mark_all_process_dead(pid, path=None):
+    if path is None:
+        path = os.environ.get('PROMETHEUS_MULTIPROC_DIR')
+
+    try:
+        from prometheus_client import Gauge
+        import glob
+        for mode in Gauge._MULTIPROC_MODES:
+            for f in glob.glob(os.path.join(path, f'gauge_{mode}_{pid}.db')):
+                os.remove(f)
+    except ImportError:
+        pass
+
+    try:
+        os.remove(os.path.join(path, f'counter_{pid}.db'))
+        os.remove(os.path.join(path, f'histogram_{pid}.db'))
+    except Exception:
+        pass
+
+
 project_path = os.path.dirname(os.path.abspath(__file__))
 host = os.environ.get("MAIN_SERVER_HOST", "0.0.0.0")
 port = os.environ.get("MAIN_SERVER_PORT", 8001)