From 08aaa17af7ad03700fb7d6d5421cd8574fc304bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B4=94=E4=B8=BA=E4=B9=8B?= <560397@gree.com.cn>
Date: Fri, 3 Nov 2023 17:01:13 +0800
Subject: [PATCH] Update project

---
 README.md                                     |  4 +-
 application/config/config.yaml                | 11 +++
 application/config/production_config.yaml     | 20 +++-
 application/extensions/__init__.py            |  4 +-
 application/extensions/init_elasticsearch.py  | 28 ++++++
 application/extensions/init_marshmallow.py    |  2 +-
 application/lib/__init__.py                   |  1 +
 .../lib/flask_elasticsearch/__init__.py       | 13 +++
 .../lib/flask_elasticsearch/elasticsearch.py  | 93 +++++++++++++++++++
 application/utils/__init__.py                 |  1 +
 application/utils/elasticsearch/__init__.py   | 13 +++
 .../utils/elasticsearch/elasticsearch.py      | 82 ++++++++++++++++
 application/views/user/user.py                | 21 ++++-
 requirements.txt                              |  9 +-
 14 files changed, 295 insertions(+), 7 deletions(-)
 create mode 100644 application/extensions/init_elasticsearch.py
 create mode 100644 application/lib/flask_elasticsearch/__init__.py
 create mode 100644 application/lib/flask_elasticsearch/elasticsearch.py
 create mode 100644 application/utils/elasticsearch/__init__.py
 create mode 100644 application/utils/elasticsearch/elasticsearch.py

diff --git a/README.md b/README.md
index 0ddde83..3a9f42d 100644
--- a/README.md
+++ b/README.md
@@ -17,12 +17,12 @@
   - flask_sqlAlchemy ORM 框架
 - [x] flask-cli
   - flask-cli
+- [x] Elasticsearch
+  - flask_elasticsearch
 
 #### 待实现
 - [x] Apispec
   - flask_Apispec 接口文档实现
-- [x] Elasticsearch
-  - flask_elasticsearch
 - [x] Celery
   - Celery 异步定时
 - [x] PGSQL
diff --git a/application/config/config.yaml b/application/config/config.yaml
index 6c967a5..86fc09f 100644
--- a/application/config/config.yaml
+++ b/application/config/config.yaml
@@ -7,3 +7,14 @@ MySQL:
   Host: localhost
   Port: 3306
   DB: elp
+
+Elasticsearch:
+  Host: 172.28.5.39
+  Port: 9200
+  USER:
+  Password:
+  UseSSL:
+  VerifyCerts:
+  CaCerts:
+  Index: filebeat-ctocst_router-2021.11
+  Type: _doc
diff --git a/application/config/production_config.yaml b/application/config/production_config.yaml
index 485af78..86fc09f 100644
--- a/application/config/production_config.yaml
+++ b/application/config/production_config.yaml
@@ -1,2 +1,20 @@
 System:
-  Env: public123
\ No newline at end of file
+  Env: public
+
+MySQL:
+  User: root
+  Password: localhost123
+  Host: localhost
+  Port: 3306
+  DB: elp
+
+Elasticsearch:
+  Host: 172.28.5.39
+  Port: 9200
+  USER:
+  Password:
+  UseSSL:
+  VerifyCerts:
+  CaCerts:
+  Index: filebeat-ctocst_router-2021.11
+  Type: _doc
diff --git a/application/extensions/__init__.py b/application/extensions/__init__.py
index f2d5b89..fa32e33 100644
--- a/application/extensions/__init__.py
+++ b/application/extensions/__init__.py
@@ -10,8 +10,8 @@
 # @Description :
 """
 
-from flask import Flask
 
+from flask import Flask
 from .init_config import init_config
 from .init_logger import init_logger
 from .init_sqlalchemy import init_database
@@ -19,6 +19,7 @@ from .init_bcrypt import init_bcrypt
 from .init_migrate import init_migrate
 from .init_apispec import init_apispec
 from .init_marshmallow import init_marshmallow
+from .init_elasticsearch import init_elasticsearch
 
 
 def init_plugs(app: Flask) -> None:
@@ -29,3 +30,4 @@ def init_plugs(app: Flask) -> None:
     init_migrate(app)
     init_apispec(app)
     init_marshmallow(app)
+    init_elasticsearch(app)
diff --git a/application/extensions/init_elasticsearch.py b/application/extensions/init_elasticsearch.py
new file mode 100644
index 0000000..4e5aa21
--- /dev/null
+++ b/application/extensions/init_elasticsearch.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# @Version     : Python 3.11.4
+# @Software    : Sublime Text 4
+# @Author      : StudentCWZ
+# @Email       : StudentCWZ@outlook.com
+# @Date        : 2023/10/28 18:16
+# @File        : init_elasticsearch.py
+# @Description :
+"""
+
+
+from flask import Flask
+
+from application.lib import FlaskElasticsearch
+
+es = FlaskElasticsearch()
+
+
+def init_elasticsearch(app: Flask) -> None:
+    """
+    Initialize the flask_elasticsearch extension
+
+    :param app: flask.Flask application instance
+    :return: None
+    """
+    es.init_app(app)
diff --git a/application/extensions/init_marshmallow.py b/application/extensions/init_marshmallow.py
index 8923008..796e734 100644
--- a/application/extensions/init_marshmallow.py
+++ b/application/extensions/init_marshmallow.py
@@ -24,4 +24,4 @@ def init_marshmallow(app: Flask) -> None:
     :param app: flask.Flask application instance
     :return: None
     """
-    ma.init_app(app)
\ No newline at end of file
+    ma.init_app(app)
diff --git a/application/lib/__init__.py b/application/lib/__init__.py
index 47b6820..620fe13 100644
--- a/application/lib/__init__.py
+++ b/application/lib/__init__.py
@@ -11,4 +11,5 @@
 """
 
 from .config import ConsulConfig, LocalConfig
+from .flask_elasticsearch import FlaskElasticsearch
 from .flask_loguru import FlaskLoguru
diff --git a/application/lib/flask_elasticsearch/__init__.py b/application/lib/flask_elasticsearch/__init__.py
new file mode 100644
index 0000000..ac68cfa
--- /dev/null
+++ b/application/lib/flask_elasticsearch/__init__.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# @Version     : Python 3.11.4
+# @Software    : Sublime Text 4
+# @Author      : StudentCWZ
+# @Email       : StudentCWZ@outlook.com
+# @Date        : 2023/10/28 12:19
+# @File        : __init__.py
+# @Description :
+"""
+
+from .elasticsearch import FlaskElasticsearch
diff --git a/application/lib/flask_elasticsearch/elasticsearch.py b/application/lib/flask_elasticsearch/elasticsearch.py
new file mode 100644
index 0000000..88ff271
--- /dev/null
+++ b/application/lib/flask_elasticsearch/elasticsearch.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# @Version     : Python 3.11.4
+# @Software    : Sublime Text 4
+# @Author      : StudentCWZ
+# @Email       : StudentCWZ@outlook.com
+# @Date        : 2023/10/28 12:19
+# @File        : elasticsearch.py
+# @Description :
+"""
+
+from typing import Any
+
+from elasticsearch import Elasticsearch
+from flask import current_app, Flask
+from loguru import logger
+
+
+class FlaskElasticsearch:
+    def __init__(self, app=None):
+        if app is not None:
+            self.init_app(app)
+
+    def init_app(self, app: Flask) -> None:
+        """Initialize the app"""
+        if hasattr(app, 'teardown_appcontext'):
+            app.teardown_appcontext(self.teardown)
+        else:
+            app.teardown_request(self.teardown)
+
+    @staticmethod
+    def teardown(exception):
+        """Clears the Elasticsearch connection after each request."""
+        ctx = current_app._get_current_object()
+        if hasattr(ctx, 'elasticsearch'):
+            ctx.elasticsearch = None
+        if exception is not None:
+            raise RuntimeError(exception)
+
+    def __getattr__(self, item: Any) -> Any:
+        """Lazy initialization of Elasticsearch connection on first use."""
+        ctx = current_app._get_current_object()
+        if ctx is not None:
+            if not hasattr(ctx, "elasticsearch"):
+                cfg = self._get_config()
+                ctx.elasticsearch = Elasticsearch(**cfg)
+                if ctx.elasticsearch.ping():
+                    logger.info('Connected to Elasticsearch')
+                else:
+                    logger.error('Connected to Elasticsearch')
+                    raise ConnectionError('Connected to Elasticsearch')
+            return getattr(ctx.elasticsearch, item)
+
+    @staticmethod
+    def _get_config():
+        """Retrieves Elasticsearch configuration from the current Flask application context."""
+        with current_app.app_context():
+            if current_app:
+                # Retrieve configuration from current_app.config and return it
+                host = current_app.config.Elasticsearch.Host if current_app.config.get(
+                    'Elasticsearch') is not None else 'localhost'
+
+                port = int(current_app.config.Elasticsearch.Port) if current_app.config.get(
+                    'Elasticsearch') is not None else 9200
+
+                user = current_app.config.Elasticsearch.User if current_app.config.get(
+                    'Elasticsearch') is not None else 'user'
+
+                password = current_app.config.Elasticsearch.Password if current_app.config.get(
+                    'Elasticsearch') is not None else 'password'
+
+                use_ssl = current_app.config.Elasticsearch.UseSsl == 'True' if current_app.config.get(
+                    'Elasticsearch') is not None else False
+
+                verify_certs = current_app.config.Elasticsearch.VerifyCerts == 'False' if current_app.config.get(
+                    'Elasticsearch') is not None else True
+
+                ca_certs = current_app.config.Elasticsearch.CaCerts if current_app.config.get(
+                    'Elasticsearch') is not None else None
+
+                es_config = dict(
+                    hosts=[{'host': host, 'port': port}],
+                    http_auth=None if not user else (user, password),
+                    use_ssl=use_ssl,
+                    verify_certs=verify_certs,
+                    ca_certs=ca_certs
+                )
+
+                return es_config
+            else:
+                logger.error('Attempted to access application configuration outside of application context.')
+                raise RuntimeError('Attempted to access application configuration outside of application context.')
diff --git a/application/utils/__init__.py b/application/utils/__init__.py
index 9776109..eb4d1d2 100644
--- a/application/utils/__init__.py
+++ b/application/utils/__init__.py
@@ -11,3 +11,4 @@
 """
 
 from .dsn import dsn
+from .elasticsearch import ElasticsearchUtils
diff --git a/application/utils/elasticsearch/__init__.py b/application/utils/elasticsearch/__init__.py
new file mode 100644
index 0000000..afb6616
--- /dev/null
+++ b/application/utils/elasticsearch/__init__.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# @Version     : Python 3.11.4
+# @Software    : Sublime Text 4
+# @Author      : StudentCWZ
+# @Email       : StudentCWZ@outlook.com
+# @Date        : 2023/10/28 12:19
+# @File        : __init__.py
+# @Description :
+"""
+
+from .elasticsearch import ElasticsearchUtils
diff --git a/application/utils/elasticsearch/elasticsearch.py b/application/utils/elasticsearch/elasticsearch.py
new file mode 100644
index 0000000..bf6fc41
--- /dev/null
+++ b/application/utils/elasticsearch/elasticsearch.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# @Version     : Python 3.11.4
+# @Software    : Sublime Text 4
+# @Author      : StudentCWZ
+# @Email       : StudentCWZ@outlook.com
+# @Date        : 2023/10/28 12:19
+# @File        : elasticsearch.py
+# @Description :
+"""
+
+import datetime
+
+from application.extensions.init_elasticsearch import es
+
+
+class ElasticsearchUtils:
+    @classmethod
+    def dsl(cls, _start: str, _end: str, size=10000) -> dict:
+        """
+        Setting dsl
+
+        :param _start: start time
+        :param _end: end time
+        :param size: data number
+        :return: dsl
+        """
+        _dsl = {
+            "size": size,
+            "query": {
+                "bool": {
+                    "must": [{
+                        "match_phrase": {
+                            "tag": {
+                                "query": "global"
+                            }
+                        }
+                    },
+                        {
+                        "range": {
+                            "@timestamp": {
+                                "gte": (datetime.datetime.strptime(_start, "%Y-%m-%d %H:%M:%S") -
+                                        datetime.timedelta(hours=8)).strftime("%Y-%m-%dT%H:%M:%SZ"),
+                                "lte": (datetime.datetime.strptime(_end, "%Y-%m-%d %H:%M:%S") -
+                                        datetime.timedelta(hours=8)).strftime("%Y-%m-%dT%H:%M:%SZ")
+                            }
+                        }
+                    }
+                    ],
+                    "filter": [{
+                        "match_all": {}
+                    }],
+                    "should": [],
+                    "must_not": []
+                }
+            }
+        }
+        return _dsl
+
+    @classmethod
+    def search(cls, _index: str, _dsl: dict, _scroll="5m") -> dict:
+        """
+        Search data
+
+        :param _index: index name
+        :param _dsl: dsl
+        :param _scroll: scroll time
+        :return: data after search
+        """
+        return es.search(index=_index, scroll=_scroll, body=_dsl)
+
+    @classmethod
+    def scroll_search(cls, _id,  _scroll="5m") -> dict:
+        """
+        Search data
+
+        :param _id: scroll id
+        :param _scroll: scroll time
+        :return: data after search by scroll
+        """
+        return es.scroll(scroll_id=_id, scroll=_scroll, request_timeout=30)
diff --git a/application/views/user/user.py b/application/views/user/user.py
index fd75459..9c69d6b 100644
--- a/application/views/user/user.py
+++ b/application/views/user/user.py
@@ -10,11 +10,13 @@
 # @Description :
 """
 
-from flask import Blueprint, request, jsonify
+import json
+from flask import Blueprint, current_app, request, jsonify
 from application.services import UserService
 from marshmallow import ValidationError
 
 from application.schemas import UserSchema
+from application.utils import ElasticsearchUtils
 
 user_api = Blueprint('user_api', __name__)
 
@@ -45,3 +47,20 @@ def create_user():
     email = data.email
     user = UserService.create_user(username, password, email)
     return jsonify(user_schema.dump(user)), 201
+
+
+@user_api.route('/tests', methods=['POST'])
+def test_user():
+    data = request.get_json()
+    start = data.get('start')
+    end = data.get('end')
+    dsl = ElasticsearchUtils.dsl(start, end)
+    index = current_app.config.Elasticsearch.Index
+    res = ElasticsearchUtils.search(index, dsl)
+    print(len(res))
+    data = {
+        "user_name": "libai",
+        "user_age": 18,
+    }
+    res_json = json.dumps(data)
+    return res_json, 200
diff --git a/requirements.txt b/requirements.txt
index ecab631..00e8d63 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,7 @@
 alembic==1.12.1
 annotated-types==0.6.0
 apispec==6.3.0
+autopep8==2.0.4
 bcrypt==4.0.1
 blinker==1.6.3
 certifi==2023.7.22
@@ -8,12 +9,16 @@ charset-normalizer==3.3.1
 click==8.1.7
 dnspython==2.4.2
 dynaconf==3.2.3
+elastic-transport==8.10.0
+elasticsearch==7.13.0
 email-validator==2.1.0.post1
 Flask==3.0.0
 flask-apispec==0.11.4
 Flask-Bcrypt==1.0.1
+flask-marshmallow==0.15.0
 Flask-Migrate==4.0.5
 Flask-SQLAlchemy==3.1.1
+greenlet==3.0.1
 idna==3.4
 itsdangerous==2.1.2
 Jinja2==3.1.2
@@ -21,7 +26,9 @@ loguru==0.7.2
 Mako==1.2.4
 MarkupSafe==2.1.3
 marshmallow==3.20.1
+marshmallow-sqlalchemy==0.29.0
 packaging==23.2
+pycodestyle==2.11.1
 pydantic==2.4.2
 pydantic_core==2.10.1
 PyMySQL==1.1.0
@@ -31,6 +38,6 @@ requests==2.31.0
 six==1.16.0
 SQLAlchemy==2.0.22
 typing_extensions==4.8.0
-urllib3==2.0.7
+urllib3==1.26.18
 webargs==8.3.0
 Werkzeug==3.0.1
-- 
GitLab