From e82417b5c049820dd398b44bfda57ea8f3ffc31d Mon Sep 17 00:00:00 2001 From: k4xxx Date: Wed, 10 Jan 2024 11:29:07 +0800 Subject: [PATCH 1/4] feat: v1.1.1 --- .gitignore | 24 +--------------- app.ini | 3 ++ app/__init__.py | 4 +-- app/api/views.py | 1 + build.sh | 10 +++++++ conf/__init__.py | 20 +++++++++++++ conf/dev.py | 3 +- core/common/request.py | 4 +-- core/result.py | 2 +- core/task.py | 10 +++++-- main.py | 17 +++++------ setup.py | 11 ++++++++ toneagent.service | 15 ++++++++++ toneagent.spec | 64 ++++++++++++++++++++++++++++++++++++++++++ 14 files changed, 147 insertions(+), 41 deletions(-) create mode 100644 app.ini create mode 100644 build.sh create mode 100644 setup.py create mode 100644 toneagent.service create mode 100644 toneagent.spec diff --git a/.gitignore b/.gitignore index 3774bbb..ac0b529 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,2 @@ -# Byte-compiled / optimized / DLL files -# */__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# development project config -.pydevproject .idea -*.swp -.DS_Store - -logs - -.temp -.pytest_cache - -rpmbuild -build - -app.ini -/tests/data.py +dist \ No newline at end of file diff --git a/app.ini b/app.ini new file mode 100644 index 0000000..aba8df7 --- /dev/null +++ b/app.ini @@ -0,0 +1,3 @@ +[DEFAULT] +#env = dev +env = prod \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py index 6c3b97b..24c2d0e 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -44,11 +44,11 @@ prompt_info = """ │ ▄██ ████ ██▄ │ tsn: {tsn}│ │ ██ │ mode: {mode}│ │ ██ │ proxy: {proxy}│ -│ ██ │ version: 1.1.0 │ +│ ██ │ version: 1.1.1 │ └──────────────────────┴────────────────────────────────────────────┘ """.format( port=str(CONFIG.APP_PORT).ljust(31), tsn=SERVICE_CONFIG.get('tsn').ljust(34), mode=SERVICE_CONFIG.get('mode').ljust(34), proxy=SERVICE_CONFIG.get('proxy').ljust(34) -) \ No newline at end of file +) diff --git a/app/api/views.py b/app/api/views.py index 789ea79..2d10d0e 100644 --- a/app/api/views.py +++ b/app/api/views.py @@ -29,6 +29,7 @@ async def task_view(request: Request) -> HTTPResponse: task: Task = Task(**request.json) if task.sync: res = task.execute() + Result(tid=task.tid).remove() return json( { 'SUCCESS': 'ok', diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..3c041c9 --- /dev/null +++ b/build.sh @@ -0,0 +1,10 @@ +version=$1 +yum -y install rpm-build rpmdevtools git +rpmdev-setuptree + +git clone --branch v$version https://gitee.com/k4xxx/python-tone-agent.git +mv python-tone-agent toneagent-$version +tar -czvf ./rpmbuild/SOURCES/toneagent-$version.tar.gz toneagent-$version + +cp toneagent-$version/toneagent.spec ./rpmbuild/SPECS/ +rpmbuild -ba ./rpmbuild/SPECS/toneagent.spec \ No newline at end of file diff --git a/conf/__init__.py b/conf/__init__.py index c95fa47..8b91d76 100644 --- a/conf/__init__.py +++ b/conf/__init__.py @@ -26,9 +26,23 @@ def get_settings(app_env: Optional[str] = 'prod') -> Optional[BaseSettings]: settings = getattr(EnvEnum, app_env.upper()).value return settings +def mkdir_if_not_exist() -> None: + """创建系统所需目录""" + if not os.path.exists(CONFIG.BASE_DIR): + os.makedirs(CONFIG.BASE_DIR) + if not os.path.exists(CONFIG.BASE_DIR + '/conf'): + os.makedirs(CONFIG.BASE_DIR + '/conf') + if not os.path.exists(CONFIG.BASE_DIR + '/results'): + os.makedirs(CONFIG.BASE_DIR + '/results') + if not os.path.exists(CONFIG.BASE_DIR + '/scripts'): + os.makedirs(CONFIG.BASE_DIR + '/scripts') + if not os.path.isfile(CONFIG.BASE_DIR + '/conf/toneagent.config.yaml'): + with open(CONFIG.BASE_DIR + '/conf/toneagent.config.yaml', 'w') as f: + f.write(config_content) def get_service_config() -> dict: """获取service配置""" + mkdir_if_not_exist() yaml_path: Optional[str] = os.path.join(CONFIG.BASE_DIR, 'conf') with open(f'{yaml_path}/{CONFIG.AGENT_CONFIG_FILE_NAME}', 'r', encoding="utf-8") as f: data = yaml.load(f, Loader=yaml.FullLoader) @@ -42,5 +56,11 @@ try: except configparser.NoOptionError: env = 'prod' +config_content = """ +tsn: tone20000101-0001 +mode: active +proxy: https://tone-agent.openanolis.cn +""" + CONFIG: Optional[BaseSettings] = get_settings(env) SERVICE_CONFIG: Optional[dict] = get_service_config() diff --git a/conf/dev.py b/conf/dev.py index 8afebb4..73974d1 100644 --- a/conf/dev.py +++ b/conf/dev.py @@ -9,8 +9,7 @@ class Settings(BaseSettings): # app DEBUG = True SECRET_KEY: str = "*5}nAn#4GyQVO'*qq_2AAOC\\fXsvi9;m0.-Q|#jAcS" - # BASE_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '.temp') - BASE_DIR = '/Users/foo/tone/gitee/toneagent/.temp' + BASE_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '.temp') WORKS_NUM = 1 APP_PORT = 8000 diff --git a/core/common/request.py b/core/common/request.py index 3dc02c6..ad48808 100644 --- a/core/common/request.py +++ b/core/common/request.py @@ -35,9 +35,9 @@ def request_to_proxy(api: str, method: str, data: dict = None) -> Optional[dict] data: dict = _set_sign_param(data) try: if method.lower() == 'get': - res: Response = requests.get(api, params=data) + res: Response = requests.get(api, params=data, verify=False) else: - res: Response = requests.post(api, json=data) + res: Response = requests.post(api, json=data, verify=False) return res.json() except Exception as e: logger.error(f'request to proxy error: {api} | {data} | {traceback.format_exc()}') diff --git a/core/result.py b/core/result.py index c26533c..d8108d9 100644 --- a/core/result.py +++ b/core/result.py @@ -84,7 +84,7 @@ class Result(BaseObject): res: dict = request_to_proxy(link, 'post', data=data) success = res.get('SUCCESS') == 'ok' if success: - if self.status == TaskStatus.COMPLETED: + if data.get('status') == TaskStatus.COMPLETED: self.remove() else: logger.error(f'sync result to proxy failed: {self.tid} | {res}') diff --git a/core/task.py b/core/task.py index ba08263..2f9d92f 100644 --- a/core/task.py +++ b/core/task.py @@ -1,6 +1,7 @@ # coding: utf-8 import base64 +import os import subprocess from threading import Timer from typing import Optional @@ -38,7 +39,7 @@ class Task(BaseObject): link: str = get_api_link(CONFIG.AGENT_PULL_TASK_API) data: dict = {'tsn': SERVICE_CONFIG.get('tsn')} res: dict = request_to_proxy(link, 'post', data=data) - if not res['SUCCESS']: + if not res['SUCCESS'] or res['SUCCESS'] == 'False': logger.error(f'pull task failed. result: {res}') return dict() return res['TASKS'] @@ -98,8 +99,11 @@ class Task(BaseObject): exit_code=str(p.returncode), finish_time=get_present_time() ).update() - - return Result(tid=self.tid).read() + os.remove(self.script_file) + try: + return Result(tid=self.tid).read() + except FileNotFoundError: + pass def kill(self, p: subprocess.Popen) -> None: """ diff --git a/main.py b/main.py index 9b8b3d9..e932683 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,8 @@ -# # coding: utf-8 - -import os +# coding: utf-8 +import urllib3 from sanic import Sanic +from urllib3.exceptions import InsecureRequestWarning + from app import init_app, prompt_info from core.common.enum import AgentMode @@ -9,20 +10,20 @@ from core.common.log import logger from conf import SERVICE_CONFIG from core.scheduler import run_scheduler_forever, run_by_threaded +# 忽略InsecureRequestWarning警告 +urllib3.disable_warnings(InsecureRequestWarning) def main(): app: Sanic = init_app() - logger.info(prompt_info) - if SERVICE_CONFIG.get('mode') == AgentMode.ACTIVE: run_by_threaded(run_scheduler_forever) - app.run( host='0.0.0.0', port=app.config.get('APP_PORT'), - workers=app.config.get('WORKS_NUM'), - auto_reload=True + # workers=app.config.get('WORKS_NUM'), + # auto_reload=True, + single_process=True ) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..01bad8c --- /dev/null +++ b/setup.py @@ -0,0 +1,11 @@ +from distutils.core import setup + +setup(name='toneagent', + version='1.1.1', + description='toneagent', + author='foo', + author_email='mr_foo@163.com', + url='https://gitee.com/anolis/python-tone-agent', + packages=['app','conf','core','tests'], + # data_files=[('/etc/maerd',['maerd.conf'])], +) \ No newline at end of file diff --git a/toneagent.service b/toneagent.service new file mode 100644 index 0000000..00752f4 --- /dev/null +++ b/toneagent.service @@ -0,0 +1,15 @@ +[Unit] +Description=ToneAgent Service +After=network.target + +[Service] +Type=simple +ExecStart=/usr/local/toneagent/venv/bin/python3 /usr/local/toneagent/app/main.py +PIDFile=/run/toneagent.pid +KillSignal=SIGTERM +KillMode=process +Restart=on-failure + +[Install] +WantedBy=multi-user.target + diff --git a/toneagent.spec b/toneagent.spec new file mode 100644 index 0000000..92acfa4 --- /dev/null +++ b/toneagent.spec @@ -0,0 +1,64 @@ +%define name toneagent +%define version 1.1.1 +%define app_path /usr/local/toneagent +%define python_bin python3.9 +%define release 1 +# Resolve python conflicts +%define _build_id_links none + +Summary: toneagent +Name: %{name} +Version: %{version} +Release: %{release} +License: MulanPSLv2 +Group: Development/Libraries +Prefix: %{_prefix} +#BuildArch: noarch +Source: %{name}-%{version}.tar.gz +BuildRoot: %{_tmppath}/%{name}-%{version}-build +Vendor: foo +Url: https://gitee.com/anolis/python-tone-agent + +%description +ToneAgent[python] service + +%prep +%setup -q -n %{name}-%{version} + +%build +%define debug_package %{nil} + +%install +mkdir -p ${RPM_BUILD_ROOT}%{app_path}/{app,conf,logs,results,scripts,venv} +mkdir -p ${RPM_BUILD_ROOT}/usr/lib/systemd/system/ +cp -rf %{_builddir}/%{name}-%{version}/* ${RPM_BUILD_ROOT}%{app_path}/app/ +cp -f %{_builddir}/%{name}-%{version}/%{name}.service ${RPM_BUILD_ROOT}/usr/lib/systemd/system/ + +yum install -y python39 +pip3 install --upgrade pip setuptools virtualenv +virtualenv -p %{python_bin} --copies ${RPM_BUILD_ROOT}%{app_path}/venv +#virtualenv -p %{python_bin} ${RPM_BUILD_ROOT}%{app_path}/venv +${RPM_BUILD_ROOT}%{app_path}/venv/bin/pip3 install -r ${RPM_BUILD_ROOT}%{app_path}/app/requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ + +cd ${RPM_BUILD_ROOT}%{app_path}/venv/bin && \ +rm -f activate activate.fish activate.ps1 normalizer pip3 pip3.9 wheel3 wheel3.9 && \ +rm -f activate.csh activate.nu activate_this.py pip pip-3.9 wheel wheel-3.9 sanic + +%post +systemctl daemon-reload +systemctl enable %{name}.service +systemctl start %{name}.service + +%postun +rm -rf /usr/lib/systemd/system/toneagent.service +rm -rf /usr/local/toneagent +systemctl daemon-reload + +%clean +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root) +%{app_path} +#%attr(0755, root, root) {app_path}/venv/bin +%{_prefix}/lib/systemd/system/%{name}.service -- Gitee From 4f0a77f43a227690b09686078f7e6296d1769d8b Mon Sep 17 00:00:00 2001 From: k4xxx Date: Wed, 10 Jan 2024 15:31:51 +0800 Subject: [PATCH 2/4] feat: v1.1.1 --- toneagent.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/toneagent.spec b/toneagent.spec index 92acfa4..9a2aa73 100644 --- a/toneagent.spec +++ b/toneagent.spec @@ -34,6 +34,7 @@ mkdir -p ${RPM_BUILD_ROOT}/usr/lib/systemd/system/ cp -rf %{_builddir}/%{name}-%{version}/* ${RPM_BUILD_ROOT}%{app_path}/app/ cp -f %{_builddir}/%{name}-%{version}/%{name}.service ${RPM_BUILD_ROOT}/usr/lib/systemd/system/ +yum install -y gcc python39-devel yum install -y python39 pip3 install --upgrade pip setuptools virtualenv virtualenv -p %{python_bin} --copies ${RPM_BUILD_ROOT}%{app_path}/venv -- Gitee From 48539bec90d57140c74a1891ef4fddca6e54a6f5 Mon Sep 17 00:00:00 2001 From: k4xxx Date: Wed, 10 Jan 2024 15:59:04 +0800 Subject: [PATCH 3/4] feat: add spec file for loongarch64 --- toneagent-loongarch64.spec | 64 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 toneagent-loongarch64.spec diff --git a/toneagent-loongarch64.spec b/toneagent-loongarch64.spec new file mode 100644 index 0000000..f4efaae --- /dev/null +++ b/toneagent-loongarch64.spec @@ -0,0 +1,64 @@ +%define name toneagent +%define version 1.1.1 +%define app_path /usr/local/toneagent +%define python_bin python3.10 +%define release 1 +# Resolve python conflicts +%define _build_id_links none + +Summary: toneagent +Name: %{name} +Version: %{version} +Release: %{release} +License: MulanPSLv2 +Group: Development/Libraries +Prefix: %{_prefix} +#BuildArch: noarch +Source: %{name}-%{version}.tar.gz +BuildRoot: %{_tmppath}/%{name}-%{version}-build +Vendor: foo +Url: https://gitee.com/anolis/python-tone-agent + +%description +ToneAgent[python] service + +%prep +%setup -q -n %{name}-%{version} + +%build +%define debug_package %{nil} + +%install +mkdir -p ${RPM_BUILD_ROOT}%{app_path}/{app,conf,logs,results,scripts,venv} +mkdir -p ${RPM_BUILD_ROOT}/usr/lib/systemd/system/ +cp -rf %{_builddir}/%{name}-%{version}/* ${RPM_BUILD_ROOT}%{app_path}/app/ +cp -f %{_builddir}/%{name}-%{version}/%{name}.service ${RPM_BUILD_ROOT}/usr/lib/systemd/system/ + +#yum install -y gcc python39-devel +yum install -y python310 +pip3 install --upgrade virtualenv +virtualenv -p %{python_bin} --copies ${RPM_BUILD_ROOT}%{app_path}/venv +#virtualenv -p %{python_bin} ${RPM_BUILD_ROOT}%{app_path}/venv +${RPM_BUILD_ROOT}%{app_path}/venv/bin/pip3 install -r ${RPM_BUILD_ROOT}%{app_path}/app/requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ + +cd ${RPM_BUILD_ROOT}%{app_path}/venv/bin && \ +rm -f activate activate.fish activate.ps1 normalizer pip3 pip3.10 wheel3 wheel3.10 && \ +rm -f activate.csh activate.nu activate_this.py pip pip-3.10 wheel wheel-3.10 sanic + +%post +systemctl daemon-reload +systemctl enable %{name}.service +systemctl start %{name}.service + +%postun +rm -rf /usr/lib/systemd/system/toneagent.service +rm -rf /usr/local/toneagent +systemctl daemon-reload + +%clean +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root) +%{app_path} +%{_prefix}/lib/systemd/system/%{name}.service \ No newline at end of file -- Gitee From b431ddbf7a1361f7c6318f3de484d8b27d9b57c2 Mon Sep 17 00:00:00 2001 From: k4xxx Date: Fri, 1 Mar 2024 15:01:51 +0800 Subject: [PATCH 4/4] feat: v1.1.1 --- .gitignore | 4 +++- app.ini | 4 ++-- build.sh | 10 ---------- setup.py | 11 ----------- 4 files changed, 5 insertions(+), 24 deletions(-) delete mode 100644 build.sh delete mode 100644 setup.py diff --git a/.gitignore b/.gitignore index ac0b529..3acbf30 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .idea -dist \ No newline at end of file +dist +.temp +*.pyc diff --git a/app.ini b/app.ini index aba8df7..babde4a 100644 --- a/app.ini +++ b/app.ini @@ -1,3 +1,3 @@ [DEFAULT] -#env = dev -env = prod \ No newline at end of file +env = dev +#env = prod \ No newline at end of file diff --git a/build.sh b/build.sh deleted file mode 100644 index 3c041c9..0000000 --- a/build.sh +++ /dev/null @@ -1,10 +0,0 @@ -version=$1 -yum -y install rpm-build rpmdevtools git -rpmdev-setuptree - -git clone --branch v$version https://gitee.com/k4xxx/python-tone-agent.git -mv python-tone-agent toneagent-$version -tar -czvf ./rpmbuild/SOURCES/toneagent-$version.tar.gz toneagent-$version - -cp toneagent-$version/toneagent.spec ./rpmbuild/SPECS/ -rpmbuild -ba ./rpmbuild/SPECS/toneagent.spec \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 01bad8c..0000000 --- a/setup.py +++ /dev/null @@ -1,11 +0,0 @@ -from distutils.core import setup - -setup(name='toneagent', - version='1.1.1', - description='toneagent', - author='foo', - author_email='mr_foo@163.com', - url='https://gitee.com/anolis/python-tone-agent', - packages=['app','conf','core','tests'], - # data_files=[('/etc/maerd',['maerd.conf'])], -) \ No newline at end of file -- Gitee