From 22bbdf46342f3cf70e805bc4d14a9f9ab175acef Mon Sep 17 00:00:00 2001 From: Junyan Qin Date: Tue, 7 Jan 2025 15:14:11 +0800 Subject: [PATCH] chore: upload --- uploader/upload-package.py | 135 +++++++++++++++++++++++++++++++ validator/check-pkg-paths.py | 15 ++++ validator/check-prefix.py | 20 +++++ validator/test-plugin-install.py | 89 ++++++++++++++++++++ 4 files changed, 259 insertions(+) create mode 100644 uploader/upload-package.py create mode 100644 validator/check-pkg-paths.py create mode 100644 validator/check-prefix.py create mode 100644 validator/test-plugin-install.py diff --git a/uploader/upload-package.py b/uploader/upload-package.py new file mode 100644 index 0000000..7901d66 --- /dev/null +++ b/uploader/upload-package.py @@ -0,0 +1,135 @@ +import argparse +import json +import sys +import os +import subprocess +import traceback +import requests + +MARKETPLACE_BASE_URL = "" +PLUGIN_DAEMON_PATH = "./dify-plugin" +TESTING = False + + +def main(): + global MARKETPLACE_BASE_URL + global PLUGIN_DAEMON_PATH + parser = argparse.ArgumentParser(description="Upload a package or directory to the marketplace. Choose one of the following sources: -p(--package), -d(--directory), --batch-directory") + # -p, --package + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("-p", "--package", type=str, help="The package to upload") + group.add_argument("-d", "--directory", type=str, help="The directory to package and upload") + group.add_argument("--batch-directory", type=str, help="Batch upload all directories in the given directory") + parser.add_argument("-t", "--token", type=str, required=True, help="The token to use for authentication") + parser.add_argument("-u", "--base-url", type=str, help="The base url to use for the request") + parser.add_argument("-f", "--force", action="store_true", help="Force upload the package, ignore version check") + parser.add_argument("--with-changelog", action="store_true", help="Whether to read changelog from stdin") + parser.add_argument("--plugin-daemon-path", type=str, help="The path to the plugin daemon") + parser.add_argument("--test", action="store_true", help="Indicates that this is a testing") + args = parser.parse_args() + + if args.plugin_daemon_path: + PLUGIN_DAEMON_PATH = args.plugin_daemon_path + + global TESTING + TESTING = args.test + + # if --with-changelog == true, read changelog from stdin + if args.with_changelog: + changelog = sys.stdin.read() + args.changelog = changelog.strip() + else: + args.changelog = "" + + print(json.dumps(args.__dict__, indent=2)) + + if args.base_url: + MARKETPLACE_BASE_URL = args.base_url + + if args.package: + upload_package(args.package, args.token, MARKETPLACE_BASE_URL, args.force, args.changelog) + elif args.directory: + upload_directory(args.directory, args.token, MARKETPLACE_BASE_URL, args.force, args.changelog) + elif args.batch_directory: + batch_upload_directory(args.batch_directory, args.token, MARKETPLACE_BASE_URL, args.force, args.changelog) + else: + print("No package or directory provided") + + +def upload_package(package: str, token: str, base_url: str, force: bool, changelog: str): + global TESTING + + if TESTING: + print("!!! Skip uploading package in testing") + return + + url = f"{base_url}/api/v1/plugins/inner-upload" + + payload = { + "changelog": changelog, + "forcely": 'true' if force else 'false', + } + + files = [ + ("file", (package, open(package, "rb"), "application/octet-stream")) + ] + + headers = { + "Authorization": f"Bearer {token}" + } + + resp = requests.post(url, headers=headers, data=payload, files=files) + print(resp.json()) + if resp.status_code != 200 or resp.json().get("code") != 0: + raise Exception(f"Failed to upload package: {resp.json()}") + + +def upload_directory(directory: str, token: str, base_url: str, force: bool, changelog: str): + check_plugin_daemon_command_exists() + + # delete temp.difypkg if exists + if os.path.exists("temp.difypkg"): + os.remove("temp.difypkg") + + # ./dify-plugin-daemon plugin package --out temp.difypkg + result = subprocess.run([PLUGIN_DAEMON_PATH, "plugin", "package", directory, "-o", "temp.difypkg"], capture_output=True, text=True) + print(result.stdout) + print(result.stderr) + if result.returncode != 0: + raise Exception("Failed to package the directory") + + upload_package("temp.difypkg", token, base_url, force, changelog) + + +def batch_upload_directory(directory: str, token: str, base_url: str, force: bool, changelog: str): + success_dirs = [] + failed_dirs = [] + for dir in os.listdir(directory): + path = os.path.join(directory, dir) + print(f"* Uploading directory: {path}") + try: + upload_directory(path, token, base_url, force, changelog) + success_dirs.append(path) + except Exception as e: + print(f"** Failed to upload directory: {path}") + print(traceback.format_exc()) + failed_dirs.append(path) + + if len(failed_dirs) > 0: + success_dirs_str = "\n ".join(success_dirs) + failed_dirs_str = "\n ".join(failed_dirs) + raise Exception(f"!!! Done.\nFailed directories({len(failed_dirs)}):\n {failed_dirs_str}\n\nSuccess directories({len(success_dirs)}):\n {success_dirs_str}") + else: + print("!!! Done.") + + +def check_plugin_daemon_command_exists(): + # ./daemon version + result = subprocess.run([PLUGIN_DAEMON_PATH, "version"], capture_output=True, text=True) + print(result.stdout) + if result.returncode != 0: + raise Exception("Plugin daemon command not found") + + +if __name__ == "__main__": + main() diff --git a/validator/check-pkg-paths.py b/validator/check-pkg-paths.py new file mode 100644 index 0000000..21c3723 --- /dev/null +++ b/validator/check-pkg-paths.py @@ -0,0 +1,15 @@ +import os +import json +import sys + +files: list[str] = json.loads(os.environ['PR_FILES']) + +if len(files) != 1: + print("only one file change allowed in a single PR.") + sys.exit(1) + +if not files[0]['path'].endswith(".difypkg"): + print("only .difypkg file allowed.") + sys.exit(1) + +print(files[0]['path']) \ No newline at end of file diff --git a/validator/check-prefix.py b/validator/check-prefix.py new file mode 100644 index 0000000..b246fed --- /dev/null +++ b/validator/check-prefix.py @@ -0,0 +1,20 @@ +import os +import json + +def get_prefix(file_path): + return (file_path.split('/')[0] + '/' + file_path.split('/')[1]) if len(file_path.split('/')) > 1 else file_path + +files = json.loads(os.environ['PR_FILES']) + +# only tools/ models/ extensions/ +files = [file for file in files if file['path'].startswith('tools/') or file['path'].startswith('models/') or file['path'].startswith('extensions/')] + +previous_prefix = get_prefix(files[0]['path']) +for file in files: + prefix = get_prefix(file['path']) + if prefix != previous_prefix: + print('not in same plugin directory') + import sys + sys.exit(1) + +print(previous_prefix) \ No newline at end of file diff --git a/validator/test-plugin-install.py b/validator/test-plugin-install.py new file mode 100644 index 0000000..0f2e882 --- /dev/null +++ b/validator/test-plugin-install.py @@ -0,0 +1,89 @@ +import os +import sys +import argparse +import subprocess +import threading +import time + +import requests + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--directory", type=str, required=True) + args = parser.parse_args() + + if args.directory: + print(f"Testing plugin in directory: {args.directory}") + else: + print("No directory provided") + + result = test_plugin_install(args.directory) + print(f"!!! Plugin test result: {result}") + if result != "success": + sys.exit(1) + +def test_plugin_install(directory: str) -> str: + # workdir: , exec: python3 -m main + result: str = "failed" + env = os.environ.copy() + try: + process = subprocess.Popen( + ["python3", "-m", "main"], + cwd=directory, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + semaphore = threading.Semaphore(0) + + def check_output(): + nonlocal result + while True: + output = process.stdout.readline() + if output == "" and process.poll() is not None: + print("!!! Process exited") + semaphore.release() + break + if output: + print(output.strip()) + if "Serving Flask app" in output: + print("!!! Detected 'Serving Flask app' in output, sending request to port 8080") + time.sleep(3) + response = requests.get("http://127.0.0.1:8080") + if response.status_code == 200 or response.status_code == 404: + print("!!! Request to port 8080 successful") + result = "success" + process.terminate() + semaphore.release() + else: + print("!!! Request to port 8080 failed") + result = "failed" + process.terminate() + semaphore.release() + break + + threading.Thread(target=check_output).start() + + def force_exit(): + time.sleep(20) + print("!!! Force exit after 20 seconds") + process.terminate() + semaphore.release() + + threading.Thread(target=force_exit, daemon=True).start() + + semaphore.acquire() + + process.wait() + + except Exception as e: + print(f"!!! An error occurred: {e}") + result = "failed" + + return result + +if __name__ == "__main__": + main()