diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f4f8e0fa06b1f8f6681565adfa16dabaff43cc32 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1,25 @@ +from sanic import Sanic +from app.conf import conf +from app.log import log +from app.db import db +from app.oss import alg_oss +from app.redis import redis + + +def init_app(): + app = Sanic('testlib') + conf.init_app(app, group='tone') + log.init_app(app) + db.init_app(app) + redis.init_app(app) + alg_oss.init_app(app) + init_blueprints(app) + return app + + +def init_blueprints(app): + from views import Views + app.blueprint(Views().bp) + + +app = init_app() diff --git a/app/conf.py b/app/conf.py new file mode 100644 index 0000000000000000000000000000000000000000..b21f593b6a70c0b4313eaf44a7512bc0fca88364 --- /dev/null +++ b/app/conf.py @@ -0,0 +1,73 @@ +import os +import sys + +import nacos + + +class SanicConf(object): + + def __init__(self): + pass + + def init_app(self, app, env=None, group=None): + self.config = None + if env is None: + env = os.environ.get('ENV', 'local') + if env == 'local': + file_name = 'app.properties' + from_nacos = False + else: + file_name = app.name + '.properties' + from_nacos = True + data = self.get_conf(env, file_name, from_nacos, group=group if group else app.name) + app.config.update(data) + self.config = app.config + + def get_conf(self, env, file_name, from_nacos=False, group=None): + if from_nacos: + conf = self.nacos_client(env).get_config(data_id=file_name, group=group) + else: + with open(file_name, 'r', encoding='utf-8') as f: + conf = f.read() + lines = conf.splitlines() + lines.extend(sys.argv) + lines.append(f'pid={os.getpid()}') + conf_dict = dict() + for line in lines: + if line.strip(): + if line[0] == '#': + continue + l = line.split('=', 1) + if len(l) < 2: + continue + key = l[0].strip() + value = l[1].strip() + conf_dict[key.upper()] = self.string_to_other(value) + return conf_dict + + def nacos_client(self, env): + nacos_client = None + nacos_client = nacos.NacosClient("server_address", "namespace", "ak","sk") + return nacos_client + + def string_to_other(self, data): + try: + return int(data) + except: + pass + try: + return float(data) + except: + pass + try: + if data.lower() == 'true': + return True + if data.lower() == 'false': + return False + raise 'not boolen' + except: + pass + return data + + +conf = SanicConf() diff --git a/app/db.py b/app/db.py new file mode 100644 index 0000000000000000000000000000000000000000..77cde245a4dffc68b190d48ad4e5f992a533b0ef --- /dev/null +++ b/app/db.py @@ -0,0 +1,39 @@ +from asyncio import current_task +from sqlalchemy.orm import sessionmaker +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.ext.asyncio import create_async_engine +from sqlalchemy.ext.asyncio import async_scoped_session +from sqlalchemy.ext.declarative import declarative_base + +Base = declarative_base() + + +class SanicDb(object): + + def __init__(self): + pass + + def init_app(self, app): + self.conn = None + self.engine = None + + @app.listener('before_server_start') + async def db_conn(app, loop): + engine = create_async_engine(app.config['DB_URL'], pool_size=app.config['POOL_SIZE'], + max_overflow=app.config['OVER_SIZE'], pool_recycle=app.config['RECYCLE']) + async with engine.begin() as conn: + if app.config['DROP_ALL']: + await conn.run_sync(Base.metadata.drop_all) + if app.config['CREATE_ALL']: + await conn.run_sync(Base.metadata.create_all) + async_session = sessionmaker(engine, expire_on_commit=False, class_=AsyncSession) + self.conn = async_scoped_session(async_session, scopefunc=current_task) + self.engine = engine + + @app.listener('after_server_stop') + async def db_conn(app, loop): + await self.conn.remove() + await self.engine.dispose() + + +db = SanicDb() diff --git a/app/error_hander.py b/app/error_hander.py new file mode 100644 index 0000000000000000000000000000000000000000..537968142e339cd4b98f247710b9911108f671ae --- /dev/null +++ b/app/error_hander.py @@ -0,0 +1,12 @@ +from sanic import text +from sanic.handlers import ErrorHandler + + +class CustomErrorHandler(ErrorHandler): + def default(self, request, exception): + """ handles errors that have no error handlers assigned """ + return super().default(request, exception) + + +async def server_error_handler(request, exception): + return text("Oops, server error", status=500) diff --git a/app/leveldb.py b/app/leveldb.py new file mode 100644 index 0000000000000000000000000000000000000000..5b79d7812b84d20cb061a4579eec40c1259a71e8 --- /dev/null +++ b/app/leveldb.py @@ -0,0 +1,36 @@ +import os + +import leveldb + + +class SanicLeveldb(object): + def __init__(self): + pass + + def init_app(self, app): + self.db = None + + @app.listener('before_server_start') + async def new(app, loop): + datastore_path = os.path.join(os.environ['HOME'], app.config['LEVELDB']) + if not os.path.exists(datastore_path): + os.makedirs(datastore_path) + self.db = leveldb.LevelDB(datastore_path) + + def put(self, key, value): + self.db.Put(key, value) + + def delete(self, key): + self.db.Delete(key) + + def update(self, key, value): + self.db.Put(key, value) + + def get(self, key): + return self.db.Get(key) + + def iter(self): + return self.db.RangeIter() + + +file_store = SanicLeveldb() diff --git a/app/log.py b/app/log.py new file mode 100644 index 0000000000000000000000000000000000000000..8aa0b84f68386543613f734c635bc10dfdebb117 --- /dev/null +++ b/app/log.py @@ -0,0 +1,86 @@ +import os +import logging +import logging.config + + +class SanicLog(object): + + def __init__(self): + pass + + def init_app(self, app): + self.logger = None + if not app.config['LOG_MODE']: + self.logger = logging.getLogger("sanic.root") + return + if not os.path.exists('logs'): + os.makedirs('logs') + level = "DEBUG" if app.config['DEBUG'] else "INFO" + filename = os.path.join('logs', app.name + '.log') + maxBytes = app.config['LOG_BYTES'] + backup = app.config['LOG_BACKUP'] + formatter = app.config['FORMATTER'] + formatter_acess = app.config['FORMATTER_ACCESS'] + + LOGGING_CONFIG = dict( + version=1, + disable_existing_loggers=False, + loggers={ + "sanic.root": {"level": level, "handlers": ["console"]}, + "sanic.error": { + "level": level, + "handlers": ["error_console"], + "propagate": True, + "qualname": "sanic.error", + }, + "sanic.access": { + "level": level, + "handlers": ["access_console"], + "propagate": True, + "qualname": "sanic.access", + }, + }, + handlers={ + "console": { + "class": "concurrent_log_handler.ConcurrentRotatingFileHandler", + "level": level, + "formatter": "generic", + "filename": filename, + "maxBytes": maxBytes, + "backupCount": backup + }, + "error_console": { + "class": "concurrent_log_handler.ConcurrentRotatingFileHandler", + "level": level, + "formatter": "generic", + "filename": filename, + "maxBytes": maxBytes, + "backupCount": backup + }, + "access_console": { + "class": "concurrent_log_handler.ConcurrentRotatingFileHandler", + "level": level, + "formatter": "access", + "filename": filename, + "maxBytes": maxBytes, + "backupCount": backup + }, + }, + formatters={ + "generic": { + "format": formatter, + "datefmt": "[%Y-%m-%d %H:%M:%S %z]", + "class": "logging.Formatter", + }, + "access": { + "format": formatter_acess, + "datefmt": "[%Y-%m-%d %H:%M:%S %z]", + "class": "logging.Formatter", + }, + }, + ) + logging.config.dictConfig(LOGGING_CONFIG) + self.logger = logging.getLogger("sanic.root") + + +log = SanicLog() diff --git a/app/oss.py b/app/oss.py new file mode 100644 index 0000000000000000000000000000000000000000..f8052efd6a6824472932b42051a696f17747dbb7 --- /dev/null +++ b/app/oss.py @@ -0,0 +1,77 @@ +import oss2 +from oss2.exceptions import NoSuchKey + + +class OSSClient(object): + LINK_VALID_TIME = 7200 + READABLE_FILE_LIST = ['log', 'json', 'txt', 'text', 'sh', 'yaml', 'time', 'output', 'task', 'fs', 'cvs'] + + def __init__(self): + pass + + def init_app(self, app): + self.access_id = None + self.access_key = None + self.endpoint = None + self.bucket_name = None + + @app.listener('before_server_start') + async def redis_conn(app, loop): + self.access_id = app.config['ALG_OSS_ACCESS_KEY'] + self.access_key = app.config['ALG_OSS_ACCESS_SECRET'] + self.bucket_name = app.config['ALG_OSS_BUCKET'] + self.endpoint = app.config['ALG_OSS_ENDPOINT'] + auth = oss2.Auth(self.access_id, self.access_key) + self.bucket = oss2.Bucket(auth, self.endpoint, self.bucket_name) + + def file_exists(self, file_name): + return self.bucket.object_exists(file_name) + + def put_file(self, file_name): + with open(file_name, 'rb') as file_handle: + return self.bucket.put_object(file_name, file_handle) + + def put_object_from_file(self, object_name, local_file): + return self.bucket.put_object_from_file(object_name, local_file) + + def put_object_from_bytes(self, object_name, file_bytes): + return self.bucket.put_object(object_name, file_bytes) + + def get_object(self, object_name, params=None, headers=None): + try: + return self.bucket.get_object(object_name, params=params, headers=headers) + except NoSuchKey: + return None + + def get_object_to_file(self, object_name, local_file): + try: + return self.bucket.get_object_to_file(object_name, local_file) + except NoSuchKey: + return None + + def get_sign_url(self, object_name, expires=7200): + object_name = object_name.strip('/') + if self.bucket.object_exists(object_name): + sign_url = self.bucket.sign_url('GET', object_name, expires) + if isinstance(sign_url, str): + sign_url = sign_url.replace('http://', 'https://') + sign_url = sign_url.replace('%2F', '/') + return sign_url + else: + raise RuntimeError('object:%s not exist in oss bucket:%s' % (object_name, self.bucket)) + + def update_file_content_type(self, object_name, content_type='text/plain'): + object_name = object_name.strip('/') + result = self.bucket.update_object_meta(object_name, {'Content-Type': content_type}) + return result.status == 200 + + def get_file_content_type(self, file): + file_headers = self.bucket.head_object(file) + return file_headers.headers.get('Content-Type') + + def remove_object(self, objects, params=None, headers=None): + result = self.bucket.delete_object(objects, params, headers) + return result.status == 204 + + +alg_oss = OSSClient() diff --git a/app/redis.py b/app/redis.py new file mode 100644 index 0000000000000000000000000000000000000000..b6f209492a8b37a4cf157efdb1b738d126a08a8a --- /dev/null +++ b/app/redis.py @@ -0,0 +1,21 @@ +import aioredis + + +class SanicRedis(object): + + def __init__(self): + pass + + def init_app(self, app): + self.conn = None + + @app.listener('before_server_start') + async def redis_conn(app, loop): + self.conn = aioredis.from_url(app.config['REDIS_URL']) + + @app.listener('after_server_stop') + async def redis_close(app, loop): + await self.conn.close() + + +redis = SanicRedis() diff --git a/app/setting.py b/app/setting.py new file mode 100644 index 0000000000000000000000000000000000000000..2ca032bac6ce1334c2248ba70b53e8a16a84d635 --- /dev/null +++ b/app/setting.py @@ -0,0 +1,13 @@ +import os + +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "../")) + +STATIC_URL = '/static/static/' +STATICFILES_DIRS = ( + os.path.join(BASE_DIR, 'static/'), +) +STATIC_ROOT = os.path.join(BASE_DIR, '/static/').replace('\\', '/') + +MEDIA_URL = '/static/media/' +MEDIA_ROOT = os.path.join(BASE_DIR, 'static/media/').replace('\\', '/') +OFFLINE_DATA_DIR = 'offline_data'