From 9c8b1dcef527d998c2304c501d4d878994d0a060 Mon Sep 17 00:00:00 2001 From: H Date: Wed, 23 Jul 2025 20:41:13 +0800 Subject: [PATCH] init --- common/__init__.py | 0 common/config.py | 96 +++ common/file_utils.py | 155 ++++ common/http_utils.py | 103 +++ common/logger.py | 100 +++ common/rpm.py | 95 +++ config/fedora/fedora.repo | 36 + config/repo_configs.json | 94 +++ main.py | 145 ++++ main_test.py | 75 ++ readme.md | 114 +++ repodata/__init__.py | 0 repodata/checker.py | 251 ++++++ repodata/parser.py | 519 ++++++++++++ requirements.txt | 5 + script/01_repodata_check/README.md | 27 + .../repodata_rpmlist_check_all.py | 354 ++++++++ script/02_package_signatures/README.md | 37 + script/02_package_signatures/check_pgpsig.sh | 11 + script/03_package_hash/README.md | 38 + script/03_package_hash/compare_file_hash.sh | 31 + script/04_package_tps_test/README.md | 62 ++ .../04_package_tps_test/README_compare_ldy.md | 91 +++ script/04_package_tps_test/compare-0522.py | 371 +++++++++ script/04_package_tps_test/compare.py | 319 ++++++++ script/04_package_tps_test/config.ini | 170 ++++ .../04_package_tps_test/test_srpm_up_down.py | 137 ++++ .../04_package_tps_test/test_测试项2-ldy.sh | 76 ++ script/05_src_multi_version_test/README.md | 36 + .../test_srpm_up_down_0512.py | 232 ++++++ script/06_dnf_repoclosure/README.md | 67 ++ .../test_pkg_repoclosure_aarch64.py | 143 ++++ .../test_pkg_repoclosure_loongarch64.py | 96 +++ .../test_pkg_repoclosure_mips64.py | 89 ++ .../test_pkg_repoclosure_x86_64.py | 132 +++ script/07_compat_repo_check/README.md | 22 + script/07_compat_repo_check/tools/compat.py | 179 +++++ script/07_compat_repo_check/tools/repolist | 77 ++ .../07_compat_repo_check/tools/requirements | 8 + .../08_V10SPX_products_upgrade_test/.gitkeep | 0 .../08_V10SPX_products_upgrade_test/README.md | 22 + .../upgrade_tool_pri_0512.py | 465 +++++++++++ script/09_v10spx_repo-rpm_cmp/README.md | 21 + .../check_update_prod_final.py | 350 ++++++++ .../repo_cmp_20250103.py | 759 ++++++++++++++++++ script/09_v10spx_repo-rpm_cmp/repodata.py | 453 +++++++++++ script/10_black_check/README.md | 18 + .../check_black_rpms_in_repos.py | 119 +++ script/10_black_check/repository_config.json | 124 +++ 49 files changed, 6924 insertions(+) create mode 100644 common/__init__.py create mode 100644 common/config.py create mode 100644 common/file_utils.py create mode 100644 common/http_utils.py create mode 100644 common/logger.py create mode 100644 common/rpm.py create mode 100644 config/fedora/fedora.repo create mode 100644 config/repo_configs.json create mode 100644 main.py create mode 100644 main_test.py create mode 100644 readme.md create mode 100644 repodata/__init__.py create mode 100644 repodata/checker.py create mode 100644 repodata/parser.py create mode 100644 requirements.txt create mode 100644 script/01_repodata_check/README.md create mode 100644 script/01_repodata_check/repodata_rpmlist_check_all.py create mode 100644 script/02_package_signatures/README.md create mode 100644 script/02_package_signatures/check_pgpsig.sh create mode 100644 script/03_package_hash/README.md create mode 100644 script/03_package_hash/compare_file_hash.sh create mode 100644 script/04_package_tps_test/README.md create mode 100644 script/04_package_tps_test/README_compare_ldy.md create mode 100644 script/04_package_tps_test/compare-0522.py create mode 100644 script/04_package_tps_test/compare.py create mode 100644 script/04_package_tps_test/config.ini create mode 100644 script/04_package_tps_test/test_srpm_up_down.py create mode 100644 script/04_package_tps_test/test_测试项2-ldy.sh create mode 100644 script/05_src_multi_version_test/README.md create mode 100644 script/05_src_multi_version_test/test_srpm_up_down_0512.py create mode 100644 script/06_dnf_repoclosure/README.md create mode 100644 script/06_dnf_repoclosure/test_pkg_repoclosure_aarch64.py create mode 100644 script/06_dnf_repoclosure/test_pkg_repoclosure_loongarch64.py create mode 100644 script/06_dnf_repoclosure/test_pkg_repoclosure_mips64.py create mode 100644 script/06_dnf_repoclosure/test_pkg_repoclosure_x86_64.py create mode 100644 script/07_compat_repo_check/README.md create mode 100644 script/07_compat_repo_check/tools/compat.py create mode 100644 script/07_compat_repo_check/tools/repolist create mode 100644 script/07_compat_repo_check/tools/requirements create mode 100644 script/08_V10SPX_products_upgrade_test/.gitkeep create mode 100644 script/08_V10SPX_products_upgrade_test/README.md create mode 100644 script/08_V10SPX_products_upgrade_test/upgrade_tool_pri_0512.py create mode 100644 script/09_v10spx_repo-rpm_cmp/README.md create mode 100644 script/09_v10spx_repo-rpm_cmp/check_update_prod_final.py create mode 100644 script/09_v10spx_repo-rpm_cmp/repo_cmp_20250103.py create mode 100644 script/09_v10spx_repo-rpm_cmp/repodata.py create mode 100644 script/10_black_check/README.md create mode 100644 script/10_black_check/check_black_rpms_in_repos.py create mode 100644 script/10_black_check/repository_config.json diff --git a/common/__init__.py b/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/common/config.py b/common/config.py new file mode 100644 index 0000000..bfaa947 --- /dev/null +++ b/common/config.py @@ -0,0 +1,96 @@ +# -*- coding:utf-8 -*- +# 配置管理模块 + +import json +import os +import yaml +from typing import Dict, List, Any, Optional + +class ConfigManager: + """配置管理类，负责加载和管理各种配置""" + + @staticmethod + def load_json_config(config_path: str) -> Dict: + """ + 加载JSON格式的配置文件 + + Args: + config_path: JSON配置文件路径 + + Returns: + 加载的配置字典 + """ + try: + with open(config_path, 'r', encoding='utf-8') as f: + return json.load(f) + except Exception as e: + print(f"加载配置文件 {config_path} 失败: {str(e)}") + return {} + + @staticmethod + def load_yaml_config(config_path: str) -> Dict: + """ + 加载YAML格式的配置文件 + + Args: + config_path: YAML配置文件路径 + + Returns: + 加载的配置字典 + """ + try: + with open(config_path, 'r', encoding='utf-8') as f: + return yaml.safe_load(f) + except Exception as e: + print(f"加载配置文件 {config_path} 失败: {str(e)}") + return {} + + @staticmethod + def save_json_config(config_path: str, config_data: Dict) -> bool: + """ + 保存配置到JSON文件 + + Args: + config_path: 目标JSON文件路径 + config_data: 要保存的配置数据 + + Returns: + 是否保存成功 + """ + try: + os.makedirs(os.path.dirname(config_path), exist_ok=True) + with open(config_path, 'w', encoding='utf-8') as f: + json.dump(config_data, f, indent=4, ensure_ascii=False) + return True + except Exception as e: + print(f"保存配置到 {config_path} 失败: {str(e)}") + return False + +# 默认仓库配置 +DEFAULT_REPO_CONFIGS = [ + { + "product": "V10-SP3-2403", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/V10SP3-2403/os/adv/lic/", + "architectures": ["x86_64", "aarch64", "loongarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-SP3", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/V10SP3/os/adv/lic/", + "architectures": ["x86_64", "aarch64", "loongarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-SP2", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/V10SP2/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-SP1.1", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/", + "architectures": ["x86_64", "aarch64", "mips64el", "loongarch64"], + "repo_types": ["updates"] + }, + # 其他仓库配置可以通过配置文件加载... +] diff --git a/common/file_utils.py b/common/file_utils.py new file mode 100644 index 0000000..7808718 --- /dev/null +++ b/common/file_utils.py @@ -0,0 +1,155 @@ +# -*- coding:utf-8 -*- +# 文件处理工具模块 + +import os +import bz2 +import gzip +import lzma +import shutil +import logging +from typing import Optional, List, Tuple, Union, IO + +# 配置日志 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger("file_utils") + +class FileUtils: + """文件处理工具类，处理文件操作和压缩文件""" + + @staticmethod + def ensure_dir_exists(directory: str) -> bool: + """ + 确保目录存在，如果不存在则创建 + + Args: + directory: 目录路径 + + Returns: + 是否成功创建或已存在 + """ + try: + if not os.path.exists(directory): + os.makedirs(directory) + return True + except Exception as e: + logger.error(f"创建目录失败: {directory}, 错误: {str(e)}") + return False + + @staticmethod + def extract_bz2(source_path: str, target_path: str) -> bool: + """ + 解压BZ2文件 + + Args: + source_path: BZ2文件路径 + target_path: 解压后文件路径 + + Returns: + 是否解压成功 + """ + try: + logger.info(f"解压BZ2文件: {source_path} -> {target_path}") + with bz2.BZ2File(source_path) as bz2_file: + data = bz2_file.read() + + with open(target_path, 'wb') as f: + f.write(data) + + return True + except Exception as e: + logger.error(f"解压BZ2文件失败: {source_path}, 错误: {str(e)}") + return False + + @staticmethod + def extract_xz(source_path: str, target_path: str) -> bool: + """ + 解压XZ文件 + + Args: + source_path: XZ文件路径 + target_path: 解压后文件路径 + + Returns: + 是否解压成功 + """ + try: + logger.info(f"解压XZ文件: {source_path} -> {target_path}") + with lzma.open(source_path, 'rb') as input_file: + with open(target_path, 'wb') as output_file: + shutil.copyfileobj(input_file, output_file) + + return True + except Exception as e: + logger.error(f"解压XZ文件失败: {source_path}, 错误: {str(e)}") + return False + + @staticmethod + def extract_gz(source_path: str, target_path: str) -> bool: + """ + 解压GZ文件 + + Args: + source_path: GZ文件路径 + target_path: 解压后文件路径 + + Returns: + 是否解压成功 + """ + try: + logger.info(f"解压GZ文件: {source_path} -> {target_path}") + with gzip.open(source_path, 'rb') as input_file: + with open(target_path, 'wb') as output_file: + shutil.copyfileobj(input_file, output_file) + + return True + except Exception as e: + logger.error(f"解压GZ文件失败: {source_path}, 错误: {str(e)}") + return False + + @staticmethod + def write_to_file(file_path: str, content: str, mode: str = 'w', encoding: str = 'utf-8') -> bool: + """ + 写入内容到文件 + + Args: + file_path: 文件路径 + content: 文件内容 + mode: 文件打开模式 + encoding: 编码方式 + + Returns: + 是否写入成功 + """ + try: + # 确保目录存在 + directory = os.path.dirname(file_path) + if directory and not os.path.exists(directory): + os.makedirs(directory) + + with open(file_path, mode, encoding=encoding) as f: + f.write(content) + + return True + except Exception as e: + logger.error(f"写入文件失败: {file_path}, 错误: {str(e)}") + return False + + @staticmethod + def read_file(file_path: str, mode: str = 'r', encoding: Optional[str] = 'utf-8') -> Optional[Union[str, bytes]]: + """ + 读取文件内容 + + Args: + file_path: 文件路径 + mode: 文件打开模式 + encoding: 编码方式（二进制模式下为None） + + Returns: + 文件内容，失败返回None + """ + try: + with open(file_path, mode, encoding=encoding) as f: + return f.read() + except Exception as e: + logger.error(f"读取文件失败: {file_path}, 错误: {str(e)}") + return None diff --git a/common/http_utils.py b/common/http_utils.py new file mode 100644 index 0000000..207480c --- /dev/null +++ b/common/http_utils.py @@ -0,0 +1,103 @@ +# -*- coding:utf-8 -*- +# HTTP工具模块 + +import requests +import logging +from typing import Dict, Optional, Union, Any +from urllib.parse import urljoin + +# 配置日志 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger("http_utils") + +class HttpClient: + """HTTP客户端工具类，处理常见的HTTP请求操作""" + + DEFAULT_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" + } + + @staticmethod + def get(url: str, headers: Optional[Dict[str, str]] = None, + params: Optional[Dict[str, Any]] = None, timeout: int = 30, + verify: bool = False) -> Optional[requests.Response]: + """ + 发送GET请求 + + Args: + url: 请求的URL + headers: 请求头 + params: 查询参数 + timeout: 超时时间（秒） + verify: 是否验证SSL证书 + + Returns: + Response对象，请求失败返回None + """ + _headers = headers or HttpClient.DEFAULT_HEADERS + + try: + logger.debug(f"发送GET请求: {url}") + response = requests.get(url, headers=_headers, params=params, + timeout=timeout, verify=verify) + + if response.status_code != 200: + logger.warning(f"请求失败，状态码：{response.status_code}, URL: {url}") + + return response + except Exception as e: + logger.error(f"请求异常: {url}, 错误: {str(e)}") + return None + + @staticmethod + def download_file(url: str, save_path: str, headers: Optional[Dict[str, str]] = None, + timeout: int = 60, verify: bool = False, chunk_size: int = 8192) -> bool: + """ + 下载文件 + + Args: + url: 文件URL + save_path: 保存路径 + headers: 请求头 + timeout: 超时时间（秒） + verify: 是否验证SSL证书 + chunk_size: 每次读取的块大小 + + Returns: + 是否下载成功 + """ + _headers = headers or HttpClient.DEFAULT_HEADERS + + try: + logger.info(f"正在下载: {url} -> {save_path}") + response = requests.get(url, headers=_headers, timeout=timeout, + verify=verify, stream=True) + + if response.status_code != 200: + logger.error(f"下载失败，状态码: {response.status_code}") + return False + + with open(save_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=chunk_size): + if chunk: + f.write(chunk) + + logger.info(f"下载完成: {save_path}") + return True + except Exception as e: + logger.error(f"下载文件异常: {url}, 错误: {str(e)}") + return False + + @staticmethod + def join_url(base_url: str, path: str) -> str: + """ + 拼接URL + + Args: + base_url: 基础URL + path: 路径 + + Returns: + 完整URL + """ + return urljoin(base_url.rstrip('/') + '/', path.lstrip('/')) diff --git a/common/logger.py b/common/logger.py new file mode 100644 index 0000000..8cb1a68 --- /dev/null +++ b/common/logger.py @@ -0,0 +1,100 @@ +# -*- coding:utf-8 -*- +# 日志管理模块 + +import os +import logging +from typing import Optional +from datetime import datetime + +class LoggerManager: + """日志管理类，用于配置和创建日志器""" + + @staticmethod + def setup_logger(name: str, log_file: Optional[str] = None, + level: int = logging.INFO, + format_str: str = '%(asctime)s - %(name)s - %(levelname)s - %(message)s', + console_output: bool = True) -> logging.Logger: + """ + 设置并返回一个日志器 + + Args: + name: 日志器名称 + log_file: 日志文件路径，None表示不输出到文件 + level: 日志级别 + format_str: 日志格式 + console_output: 是否输出到控制台 + + Returns: + 配置好的日志器 + """ + # 创建日志器 + logger = logging.getLogger(name) + logger.setLevel(level) + + # 创建格式化器 + formatter = logging.Formatter(format_str) + + # 如果指定了日志文件 + if log_file: + # 确保日志目录存在 + log_dir = os.path.dirname(log_file) + if log_dir and not os.path.exists(log_dir): + os.makedirs(log_dir) + + # 创建文件处理器 + file_handler = logging.FileHandler(log_file) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + # 如果需要输出到控制台 + if console_output: + # 创建控制台处理器 + console_handler = logging.StreamHandler() + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + + return logger + + @staticmethod + def get_timestamped_log_path(base_dir: str, prefix: str, ext: str = '.log') -> str: + """ + 生成带时间戳的日志文件路径 + + Args: + base_dir: 基础目录 + prefix: 文件名前缀 + ext: 文件扩展名 + + Returns: + 完整的日志文件路径 + """ + # 确保基础目录存在 + if not os.path.exists(base_dir): + os.makedirs(base_dir) + + # 生成时间戳 + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + + # 构建文件名 + filename = f"{prefix}_{timestamp}{ext}" + + return os.path.join(base_dir, filename) + +# 创建一个全局日志器 +def get_logger(name: str = 'repo_test', log_file: Optional[str] = None, + level: int = logging.INFO) -> logging.Logger: + """ + 获取全局日志器 + + Args: + name: 日志器名称 + log_file: 日志文件路径 + level: 日志级别 + + Returns: + 日志器 + """ + return LoggerManager.setup_logger(name, log_file, level) + +# 默认日志器 +default_logger = get_logger() diff --git a/common/rpm.py b/common/rpm.py new file mode 100644 index 0000000..67238d0 --- /dev/null +++ b/common/rpm.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + + +import os +import glob +import xml.etree.ElementTree as ET +import tempfile +import requests + +class YumRepoParser: + def __init__(self, config_dir='/etc/yum.repo.d/', config_files=None): + self.config_dir = config_dir + self.config_files = config_files or self._discover_config_files() + + def _discover_config_files(self): + """ + 自动发现配置文件（*.repo） + """ + pattern = os.path.join(self.config_dir, '*.repo') + return glob.glob(pattern) + + def parse_repo_metadata(self, repo_path_or_url): + """ + 解析指定 yum 仓库的 repodata/primary.xml，支持本地路径和远程 http(s) 地址 + 返回 src 包和二进制包列表 + """ + if repo_path_or_url.startswith('http://') or repo_path_or_url.startswith('https://'): + primary_xml_url = repo_path_or_url.rstrip('/') + '/repodata/primary.xml' + with tempfile.NamedTemporaryFile(delete=False) as tmp: + try: + resp = requests.get(primary_xml_url, timeout=30) + resp.raise_for_status() + tmp.write(resp.content) + tmp_path = tmp.name + except Exception as e: + raise RuntimeError(f"下载远程 primary.xml 失败: {primary_xml_url}, 原因: {e}") + else: + tmp_path = os.path.join(repo_path_or_url, 'repodata', 'primary.xml') + if not os.path.exists(tmp_path): + raise FileNotFoundError(f"未找到 primary.xml： {repo_path_or_url}") + + src_packages = [] + bin_packages = [] + + tree = ET.parse(tmp_path) + root = tree.getroot() + + for package in root.findall('.//package'): + pkg_type = package.get('type') + name = package.findtext('name') + location = package.find('location').get('href') + if pkg_type == 'src': + src_packages.append({'name': name, 'location': location}) + else: + bin_packages.append({'name': name, 'location': location}) + + # 清理临时文件（远程下载时） + if repo_path_or_url.startswith('http://') or repo_path_or_url.startswith('https://'): + os.remove(tmp_path) + + return src_packages, bin_packages + + def get_all_repos(self): + """ + 解析所有配置文件，返回所有仓库地址列表，支持 baseurl/mirrorlist/metalink + """ + repos = [] + for cfg in self.config_files: + with open(cfg, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line.startswith('baseurl='): + url = line.split('=', 1)[1].strip() + repos.append(url) + elif line.startswith('mirrorlist='): + url = line.split('=', 1)[1].strip() + repos.append(url) + elif line.startswith('metalink='): + url = line.split('=', 1)[1].strip() + repos.append(url) + return repos + +# 示例用法 +if __name__ == "__main__": + parser = YumRepoParser() + repo_urls = parser.get_all_repos() + for repo_url in repo_urls: + # 假设 repo_url 是本地路径，实际可根据需求下载 repodata/primary.xml + try: + src_pkgs, bin_pkgs = parser.parse_repo_metadata(repo_url) + print(f"仓库: {repo_url}") + print(f"源码包: {len(src_pkgs)}，二进制包: {len(bin_pkgs)}") + except Exception as e: + print(f"解析失败: {repo_url}，原因: {e}") \ No newline at end of file diff --git a/config/fedora/fedora.repo b/config/fedora/fedora.repo new file mode 100644 index 0000000..67df4a8 --- /dev/null +++ b/config/fedora/fedora.repo @@ -0,0 +1,36 @@ +[fedora] +name=Fedora $releasever - $basearch +baseurl=https://mirrors.tuna.tsinghua.edu.cn/fedora/releases/$releasever/Everything/$basearch/os/ +#metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch +enabled=1 +countme=1 +metadata_expire=7d +repo_gpgcheck=0 +type=rpm +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch +skip_if_unavailable=False + +[fedora-debuginfo] +name=Fedora $releasever - $basearch - Debug +baseurl=https://mirrors.tuna.tsinghua.edu.cn/fedora/updates/$releasever/Everything/$basearch/ +#metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-debug-$releasever&arch=$basearch +enabled=0 +metadata_expire=7d +repo_gpgcheck=0 +type=rpm +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch +skip_if_unavailable=False + +[fedora-source] +name=Fedora $releasever - Source +#baseurl=https://mirrors.tuna.tsinghua.edu.cn/fedora/updates/$releasever/Everything/$basearch/ +metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-source-$releasever&arch=$basearch +enabled=0 +metadata_expire=7d +repo_gpgcheck=0 +type=rpm +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch +skip_if_unavailable=False \ No newline at end of file diff --git a/config/repo_configs.json b/config/repo_configs.json new file mode 100644 index 0000000..2565c72 --- /dev/null +++ b/config/repo_configs.json @@ -0,0 +1,94 @@ +{ + "repo_configs": [ + { + "product": "V10-SP3-2403", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/V10SP3-2403/os/adv/lic/", + "architectures": ["x86_64", "aarch64", "loongarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-SP3", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/V10SP3/os/adv/lic/", + "architectures": ["x86_64", "aarch64", "loongarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-SP2", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/V10SP2/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-SP1.1", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/", + "architectures": ["x86_64", "aarch64", "mips64el", "loongarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-HPC", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/HPC/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-ZJ", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/V10-ZJ/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-2309A", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/2309A/os/adv/lic/", + "architectures": ["aarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-2309B", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/2309B/os/adv/lic/", + "architectures": ["aarch64"], + "repo_types": ["updates"] + }, + { + "product": "HostOS", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/HOST/SP3/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["updates"] + }, + { + "product": "Host-2309", + "base_url": "https://update.cs2c.com.cn/private_test/repo/HOST/2309/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["updates"] + }, + { + "product": "Host-2406", + "base_url": "https://update.cs2c.com.cn/private_test/repo/HOST/2406/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["updates"] + }, + { + "product": "V7.9", + "base_url": "https://update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["updates"] + }, + { + "product": "V6.10", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V6/V6.10/os/lic/", + "architectures": ["x86_64"], + "repo_types": ["updates"] + }, + { + "product": "8U2", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/8U2/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["BaseOS-updates", "AppStream-updates", "PowerTools-updates", "Plus-updates"] + }, + { + "product": "8U8", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/8U8/os/adv/lic/", + "architectures": ["x86_64"], + "repo_types": ["BaseOS-updates", "AppStream-updates", "BaseOS", "AppStream", "kernel418", "kernel419"] + } + ] +} diff --git a/main.py b/main.py new file mode 100644 index 0000000..b79c032 --- /dev/null +++ b/main.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# 主执行文件 + +import os +import sys +import argparse +import logging +import json +from typing import Dict, List, Any + +from common.logger import get_logger, LoggerManager +from common.config import ConfigManager, DEFAULT_REPO_CONFIGS +from repodata.checker import RepoChecker + +# 设置基本日志级别 +logging.basicConfig(level=logging.INFO) +logger = get_logger("main") + +def parse_arguments(): + parser = argparse.ArgumentParser(description='仓库测试检查工具') + + parser.add_argument('--config', type=str, help='配置文件路径') + parser.add_argument('--output-dir', type=str, default='./results', help='输出目录') + parser.add_argument('--max-workers', type=int, default=5, help='最大并行检查线程数') + parser.add_argument('--product', type=str, help='仅检查指定产品') + parser.add_argument('--arch', type=str, help='仅检查指定架构') + parser.add_argument('--verbose', action='store_true', help='显示详细日志') + + return parser.parse_args() + +def load_config(config_path: str) -> List[Dict[str, Any]]: + """ + 加载配置文件 + + Args: + config_path: 配置文件路径 + + Returns: + 仓库配置列表 + """ + # 检查文件是否存在 + if not os.path.exists(config_path): + logger.error(f"配置文件不存在: {config_path}") + return DEFAULT_REPO_CONFIGS + + # 根据文件扩展名决定加载方式 + ext = os.path.splitext(config_path)[1].lower() + + if ext == '.json': + config = ConfigManager.load_json_config(config_path) + elif ext in ['.yaml', '.yml']: + config = ConfigManager.load_yaml_config(config_path) + else: + logger.error(f"不支持的配置文件格式: {ext}") + return DEFAULT_REPO_CONFIGS + + # 检查配置是否有效 + if not config or not isinstance(config, list): + logger.error("无效的配置格式，使用默认配置") + return DEFAULT_REPO_CONFIGS + + return config + +def filter_configs(configs: List[Dict[str, Any]], product: str = None, arch: str = None) -> List[Dict[str, Any]]: + """ + 根据条件过滤仓库配置 + + Args: + configs: 原始配置列表 + product: 产品名称过滤 + arch: 架构过滤 + + Returns: + 过滤后的配置列表 + """ + if not product and not arch: + return configs + + filtered_configs = [] + + for config in configs: + # 如果指定了产品，但不匹配，则跳过 + if product and config["product"] != product: + continue + + # 如果指定了架构，需要检查 + if arch: + # 克隆配置但只保留指定架构 + if arch in config["architectures"]: + new_config = config.copy() + new_config["architectures"] = [arch] + filtered_configs.append(new_config) + else: + # 没有指定架构，保留所有 + filtered_configs.append(config) + + return filtered_configs + +def setup_logging(verbose: bool): + """设置日志级别""" + level = logging.DEBUG if verbose else logging.INFO + logging.getLogger().setLevel(level) + +def main(): + """主函数""" + # 解析命令行参数 + args = parse_arguments() + + # 设置日志级别 + setup_logging(args.verbose) + + # 加载配置 + if args.config: + logger.info(f"加载配置文件: {args.config}") + repo_configs = load_config(args.config) + else: + logger.info("使用默认配置") + repo_configs = DEFAULT_REPO_CONFIGS + + # 过滤配置 + filtered_configs = filter_configs(repo_configs, args.product, args.arch) + + if not filtered_configs: + logger.error("没有匹配的仓库配置") + return 1 + + logger.info(f"将检查 {len(filtered_configs)} 个仓库配置") + + # 创建输出目录 + if not os.path.exists(args.output_dir): + os.makedirs(args.output_dir) + + # 创建检查器并执行检查 + checker = RepoChecker(args.output_dir) + results = checker.check_all_repositories(filtered_configs, args.max_workers) + + # 检查是否有失败 + has_failures = any(not result.get("passed", False) for result in results.values()) + + # 返回状态码 + return 1 if has_failures else 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/main_test.py b/main_test.py new file mode 100644 index 0000000..84b3863 --- /dev/null +++ b/main_test.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# 主执行文件 + +import os +import sys +import argparse +import platform +import logging + +from common.logger import get_logger +from common.config import ConfigManager, DEFAULT_REPO_CONFIGS +from repodata.checker import RepoChecker + +logger = get_logger("main") + +def detect_system_version(): + # 示例：根据实际需求完善 + sys_info = platform.uname() + # 假设根据发行版和版本号拼接配置文件名 + distro = getattr(sys_info, "system", "unknown").lower() + version = getattr(sys_info, "release", "unknown") + config_path = f"config/{distro}_{version}.json" + if os.path.exists(config_path): + return config_path + # 默认配置 + return "config/repo_configs.json" + +def parse_arguments(): + parser = argparse.ArgumentParser(description='仓库测试检查工具') + parser.add_argument('--config', type=str, help='配置文件路径') + parser.add_argument('--output-dir', type=str, default='./results', help='输出目录') + parser.add_argument('--max-workers', type=int, default=5, help='最大并行检查线程数') + parser.add_argument('--product', type=str, help='仅检查指定产品') + parser.add_argument('--arch', type=str, help='仅检查指定架构') + parser.add_argument('--diff', nargs=2, metavar=('A', 'B'), help='对比两个仓库（名称或链接）') + parser.add_argument('--verbose', action='store_true', help='显示详细日志') + return parser.parse_args() + +def main(): + args = parse_arguments() + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) + + # 配置文件选择逻辑 + if args.config: + config_path = args.config + else: + config_path = detect_system_version() + logger.info(f"自动检测配置文件: {config_path}") + + # 加载配置 + if os.path.exists(config_path): + repo_configs = ConfigManager.load_json_config(config_path) + else: + logger.warning("未找到配置文件，使用默认配置") + repo_configs = DEFAULT_REPO_CONFIGS + + # 过滤配置 + # ...根据 args.product 和 args.arch 过滤 repo_configs... + + # diff 模式 + if args.diff: + repo_a, repo_b = args.diff + # ...实现仓库对比逻辑... + logger.info(f"对比仓库: {repo_a} vs {repo_b}") + # diff 结果输出 + return 0 + + # 常规检查模式 + checker = RepoChecker(args.output_dir) + # ...执行检查逻辑... + return 0 + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..bc70268 --- /dev/null +++ b/readme.md @@ -0,0 +1,114 @@ +# 仓库测试工具套件 + +## 项目概述 + +本项目是一个综合性的仓库测试工具套件，用于测试和验证软件仓库的各种特性和功能。它提供了一系列工具来检查仓库数据的一致性、软件包签名、哈希值、依赖关系等。 + +## 架构设计 + +项目已重构为模块化架构，将功能划分为不同的模块，便于维护和扩展。 + +### 目录结构 + +``` +repo_test/ +├── script/ +│ └── 测试项 #测试脚本 +│ +├── common/ # 通用工具模块 +│ ├── __init__.py +│ ├── config.py # 配置管理 +│ ├── file_utils.py # 文件处理工具 +│ ├── http_utils.py # HTTP请求工具 +│ └── logger.py # 日志管理 +├── config/ # 配置文件目录 +│ └── repo_configs.json # 仓库配置 +├── repodata/ # 仓库数据检查模块 +│ ├── __init__.py +│ ├── checker.py # 仓库检查器 +│ └── parser.py # 仓库数据解析 +├── main.py # 主执行文件 +├── requirements.txt # 依赖列表 +└── README.md # 项目说明 +``` + +## 测试项 + +- 仓库repodata数据校验 +- 软件包签名 +- 软件包哈希值 +- 黑名单二次校验 +- 单个软件包安装、卸载 +- 源码包下所有二进制包升降级 +- 软件包依赖检查 +- 软件包兼容性对比 +- v10spx 产品升级 +- v10spx 仓库软件包对比 + +## 功能模块 + +### 1. 仓库数据检查 (repodata) + +检查仓库元数据的一致性，确保repodata中的软件包列表与实际可下载的软件包一致。 + +**用法**: + +```bash +python main.py --config config/repo_configs.json +``` + +### 2. 配置管理 (common.config) + +用于加载和管理各种配置文件，支持JSON、YAML、和yum仓库原生格式。 + +### 3. HTTP工具 (common.http_utils) + +处理HTTP请求和下载功能的工具类。 + +### 4. 文件工具 (common.file_utils) + +处理文件操作和压缩文件的工具类。 + +### 5. 日志管理 (common.logger) + +提供统一的日志记录功能。 + +## 安装依赖 + +```bash +pip install -r requirements.txt +``` + +## 命令行参数 + +主程序支持以下命令行参数： + +- `--config`: 配置文件路径 +- `--output-dir`: 输出目录（默认: ./results） +- `--max-workers`: 最大并行检查线程数（默认: 5） +- `--product`: 仅检查指定产品 +- `--arch`: 仅检查指定架构 +- `--verbose`: 显示详细日志 + +## 配置文件格式 + +配置文件支持JSON和YAML格式，示例如下： + +```json +{ + "repo_configs": [ + { + "product": "V10-SP3", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/V10SP3/os/adv/lic/", + "architectures": ["x86_64", "aarch64", "loongarch64"], + "repo_types": ["updates"] + } + ] +} +``` + +## 输出文件 + +- `repo_rpmlist_check_results.log`: 完整测试结果 +- `repo_rpmlist_check_fail.log`: 仅包含失败的测试结果 +- `error_*.log`: 各个仓库的详细错误信息 diff --git a/repodata/__init__.py b/repodata/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/repodata/checker.py b/repodata/checker.py new file mode 100644 index 0000000..90e80aa --- /dev/null +++ b/repodata/checker.py @@ -0,0 +1,251 @@ +# -*- coding:utf-8 -*- +# 仓库检查器模块 + +import os +import logging +from typing import List, Dict, Any, Optional, Tuple +import concurrent.futures +import time + +from common.logger import get_logger +from common.config import ConfigManager +from repodata.parser import RepoDataParser + +# 获取日志记录器 +logger = get_logger(__name__) + +class RepoChecker: + """仓库检查器，用于检查仓库数据的一致性""" + + def __init__(self, output_dir: str = "./results"): + """ + 初始化检查器 + + Args: + output_dir: 输出目录 + """ + self.output_dir = output_dir + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + # 初始化日志文件 + self.overall_log_file = os.path.join(output_dir, "repo_rpmlist_check_results.log") + self.fail_log_file = os.path.join(output_dir, "repo_rpmlist_check_fail.log") + + # 初始化日志文件 + self._init_log_files() + + def _init_log_files(self): + """初始化日志文件""" + # 清空或创建结果文件 + with open(self.overall_log_file, 'w', encoding='utf-8') as f: + f.write("=== 整体测试结果 === + +") + + with open(self.fail_log_file, 'w', encoding='utf-8') as f: + f.write("=== 失败记录 === + +") + + def check_repository(self, repo_config: Dict[str, Any]) -> Dict[str, Any]: + """ + 检查单个仓库配置 + + Args: + repo_config: 仓库配置 + + Returns: + 检查结果 + """ + product = repo_config["product"] + base_url = repo_config["base_url"] + + results = {} + + for arch in repo_config["architectures"]: + for repo_type in repo_config["repo_types"]: + # 构建完整的仓库URL + if "8U8" not in product: + repo_url = f"{base_url}{repo_type}/{arch}/" + else: + repo_url = f"{base_url}{repo_type}/{arch}/os/" + + # 生成输出文件名 + output_file = os.path.join(self.output_dir, f"rpm_check_{product}_{arch}_{repo_type}.txt") + + logger.info(f"开始测试: {product} {arch} {repo_type}") + logger.info(f"测试仓库地址: {repo_url}") + + try: + # 创建解析器 + parser = RepoDataParser(repo_url) + + # 获取HTML中的RPM列表 + html = parser.get_html_content() + if not html: + raise Exception("获取HTML内容失败") + + rpm_list_html = parser.parse_rpm_list_from_html(html) + + # 获取数据库中的RPM列表 + sqlite_path = parser.download_and_extract_sqlite() + if not sqlite_path: + raise Exception("获取数据库文件失败") + + rpm_list_sqlite = parser.get_rpm_list_from_sqlite(sqlite_path) + + # 比较两个列表 + comparison_result = parser.compare_rpm_lists(rpm_list_html, rpm_list_sqlite) + formatted_result = parser.format_comparison_result(comparison_result) + + # 记录结果 + test_passed = comparison_result["is_identical"] + test_key = f"{product}_{arch}_{repo_type}" + results[test_key] = { + "product": product, + "arch": arch, + "repo_type": repo_type, + "repo_url": repo_url, + "passed": test_passed, + "result": formatted_result, + "html_count": comparison_result["list1_count"], + "sqlite_count": comparison_result["list2_count"] + } + + # 输出到日志文件 + self._write_to_overall_log(product, arch, repo_type, repo_url, formatted_result) + + # 如果测试失败，记录到失败日志 + if not test_passed: + self._write_to_fail_log(product, arch, repo_type, repo_url, formatted_result) + + except Exception as e: + error_msg = f"处理仓库出错: {str(e)}" + logger.error(error_msg) + + # 记录错误信息 + test_key = f"{product}_{arch}_{repo_type}" + results[test_key] = { + "product": product, + "arch": arch, + "repo_type": repo_type, + "repo_url": repo_url, + "passed": False, + "error": error_msg + } + + # 输出到日志文件 + self._write_error_to_logs(product, arch, repo_type, repo_url, error_msg) + + return results + + def check_all_repositories(self, repo_configs: List[Dict[str, Any]], max_workers: int = 5) -> Dict[str, Any]: + """ + 并行检查所有仓库 + + Args: + repo_configs: 仓库配置列表 + max_workers: 最大并行工作线程数 + + Returns: + 所有检查结果 + """ + all_results = {} + start_time = time.time() + + # 使用线程池并行执行检查 + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + # 提交所有检查任务 + future_to_config = {executor.submit(self.check_repository, config): config for config in repo_configs} + + # 收集结果 + for future in concurrent.futures.as_completed(future_to_config): + config = future_to_config[future] + try: + result = future.result() + all_results.update(result) + except Exception as e: + logger.error(f"检查仓库 {config['product']} 时出错: {str(e)}") + + end_time = time.time() + duration = end_time - start_time + + # 记录总结信息 + total_tests = len(all_results) + passed_tests = sum(1 for result in all_results.values() if result.get("passed", False)) + + summary = f""" +====== 测试总结 ====== +总测试数: {total_tests} +通过测试: {passed_tests} +失败测试: {total_tests - passed_tests} +测试耗时: {duration:.2f} 秒 +""" + + logger.info(summary) + + # 将总结写入整体日志 + with open(self.overall_log_file, 'a', encoding='utf-8') as f: + f.write(summary) + + return all_results + + def _write_to_overall_log(self, product: str, arch: str, repo_type: str, repo_url: str, result_lines: List[str]): + """写入结果到整体日志""" + with open(self.overall_log_file, 'a', encoding='utf-8') as f: + f.write(f" +=== {product} {arch} {repo_type} === +") + f.write(f"Repository URL: {repo_url} +") + for line in result_lines: + f.write(line + ' +') + f.write(" +") + + def _write_to_fail_log(self, product: str, arch: str, repo_type: str, repo_url: str, result_lines: List[str]): + """写入结果到失败日志""" + with open(self.fail_log_file, 'a', encoding='utf-8') as f: + f.write(f" +=== {product} {arch} {repo_type} === +") + f.write(f"Repository URL: {repo_url} +") + for line in result_lines: + if "测试通过" not in line: + f.write(line + ' +') + f.write(" +") + + def _write_error_to_logs(self, product: str, arch: str, repo_type: str, repo_url: str, error_msg: str): + """写入错误信息到日志""" + # 写入到整体日志 + with open(self.overall_log_file, 'a', encoding='utf-8') as f: + f.write(f" +=== {product} {arch} {repo_type} === +") + f.write(f"Repository URL: {repo_url} +") + f.write(f"ERROR: {error_msg} + +") + + # 写入到失败日志 + with open(self.fail_log_file, 'a', encoding='utf-8') as f: + f.write(f" +=== {product} {arch} {repo_type} === +") + f.write(f"Repository URL: {repo_url} +") + f.write(f"ERROR: {error_msg} + +") + + # 写入到单独的错误日志 + error_log_file = os.path.join(self.output_dir, f"error_{product}_{arch}_{repo_type}.log") + with open(error_log_file, 'w', encoding='utf-8') as f: + f.write(f"Error processing {repo_url}: +{error_msg}") diff --git a/repodata/parser.py b/repodata/parser.py new file mode 100644 index 0000000..755e6f5 --- /dev/null +++ b/repodata/parser.py @@ -0,0 +1,519 @@ +# -*- coding:utf-8 -*- +# 仓库数据解析模块 + +import os +import sqlite3 +import logging +import xml.etree.ElementTree as ET +import time +from typing import List, Optional, Dict, Any, Tuple, Set +from bs4 import BeautifulSoup +from urllib.parse import unquote + +from common.http_utils import HttpClient +from common.file_utils import FileUtils + +# 获取日志记录器 +logger = logging.getLogger(__name__) + +# XML命名空间 +NAMESPACES = { + 'common': 'http://linux.duke.edu/metadata/common', + 'rpm': 'http://linux.duke.edu/metadata/rpm', + 'repo': 'http://linux.duke.edu/metadata/repo' +} + +class RepoDataParser: + """ + 仓库数据解析器，用于解析RPM仓库数据 + + 支持从HTML页面、SQLite数据库和repodata.xml文件中获取软件包列表 + """ + + def __init__(self, base_url: str): + """ + 初始化解析器 + + Args: + base_url: 仓库基础URL + """ + self.base_url = base_url + self.http_client = HttpClient() + self.file_utils = FileUtils() + self.cache_dir = os.path.join(os.getcwd(), ".cache") + + # 确保缓存目录存在 + if not os.path.exists(self.cache_dir): + os.makedirs(self.cache_dir) + + def get_html_content(self, path: str = "Packages/") -> Optional[str]: + """ + 获取HTML内容 + + Args: + path: 相对路径，默认为"Packages/" + + Returns: + HTML内容，失败返回None + """ + url = HttpClient.join_url(self.base_url, path) + logger.debug(f"获取HTML内容: {url}") + + response = self.http_client.get(url) + if not response: + logger.warning(f"获取HTML内容失败: {url}") + return None + + return response.text + + def parse_rpm_list_from_html(self, html: str) -> List[str]: + """ + 从HTML中解析RPM列表 + + Args: + html: HTML内容 + + Returns: + RPM列表 + """ + if not html: + logger.warning("HTML内容为空，无法解析") + return [] + + rpmlist = [] + try: + soup = BeautifulSoup(html, "html.parser") + raw_links = [item.get('href') for item in soup.find_all('a')] + + # 筛选出rpm文件并修正URL编码问题 + for link in raw_links: + if link and link.lower().endswith(".rpm"): + # 修正URL编码问题 + link = unquote(link) + rpmlist.append(link) + + logger.info(f"从HTML解析到{len(rpmlist)}个RPM文件") + return rpmlist + + except Exception as e: + logger.error(f"解析HTML内容失败: {str(e)}") + return [] + + def download_and_extract_sqlite(self) -> Optional[str]: + """ + 下载并解压数据库文件 + + Returns: + 解压后的数据库文件路径，失败返回None + """ + repodata_url = HttpClient.join_url(self.base_url, "repodata/") + response = self.http_client.get(repodata_url) + + if not response: + logger.error(f"获取repodata目录失败: {repodata_url}") + return None + + html = response.text + soup = BeautifulSoup(html, "html.parser") + links = [item.get('href') for item in soup.find_all('a')] + + # 生成缓存文件名 + cache_key = self.base_url.replace('://', '_').replace('/', '_').replace(':', '_') + sqlite_file_path = os.path.join(self.cache_dir, f"{cache_key}_primary.sqlite") + + # 如果缓存文件存在且不超过24小时，直接返回 + if os.path.exists(sqlite_file_path): + file_age = time.time() - os.path.getmtime(sqlite_file_path) + if file_age < 86400: # 24小时 = 86400秒 + logger.info(f"使用缓存的SQLite文件: {sqlite_file_path}") + return sqlite_file_path + + # 尝试各种数据库文件格式 + for link in links: + if not link: + continue + + # 处理BZ2格式 + if link.endswith("primary.sqlite.bz2"): + sqlite_url = HttpClient.join_url(repodata_url, link) + logger.info(f"发现BZ2数据库文件: {sqlite_url}") + + # 下载文件 + sqlite_bz2_path = os.path.join(self.cache_dir, f"{cache_key}_primary.sqlite.bz2") + if not self.http_client.download_file(sqlite_url, sqlite_bz2_path): + continue + + # 解压文件 + if not self.file_utils.extract_bz2(sqlite_bz2_path, sqlite_file_path): + continue + + return sqlite_file_path + + # 处理XZ格式 + elif link.endswith("primary.sqlite.xz"): + sqlite_url = HttpClient.join_url(repodata_url, link) + logger.info(f"发现XZ数据库文件: {sqlite_url}") + + # 下载文件 + sqlite_xz_path = os.path.join(self.cache_dir, f"{cache_key}_primary.sqlite.xz") + if not self.http_client.download_file(sqlite_url, sqlite_xz_path): + continue + + # 解压文件 + if not self.file_utils.extract_xz(sqlite_xz_path, sqlite_file_path): + continue + + return sqlite_file_path + + # 处理GZ格式 + elif link.endswith("primary.sqlite.gz"): + sqlite_url = HttpClient.join_url(repodata_url, link) + logger.info(f"发现GZ数据库文件: {sqlite_url}") + + # 下载文件 + sqlite_gz_path = os.path.join(self.cache_dir, f"{cache_key}_primary.sqlite.gz") + if not self.http_client.download_file(sqlite_url, sqlite_gz_path): + continue + + # 解压文件 + if not self.file_utils.extract_gz(sqlite_gz_path, sqlite_file_path): + continue + + return sqlite_file_path + + logger.error(f"未找到支持的数据库文件: {repodata_url}") + return None + + def get_rpm_list_from_sqlite(self, sqlite_path: str) -> List[str]: + """ + 从SQLite数据库获取RPM列表 + + Args: + sqlite_path: SQLite数据库文件路径 + + Returns: + RPM列表 + """ + if not os.path.exists(sqlite_path): + logger.error(f"SQLite数据库文件不存在: {sqlite_path}") + return [] + + rpmlist = [] + try: + # 连接数据库 + conn = sqlite3.connect(sqlite_path) + cursor = conn.cursor() + + # 查询软件包信息 + cursor.execute(""" + SELECT name, version, release, epoch, arch + FROM packages + ORDER BY name + """) + + # 生成RPM文件名 + for row in cursor: + name, version, release, epoch, arch = row + + # 处理epoch + epoch_str = "" if not epoch or epoch == "0" else f"{epoch}:" + + rpm = f"{name}-{epoch_str}{version}-{release}.{arch}.rpm" + rpmlist.append(rpm) + + logger.info(f"从SQLite获取到{len(rpmlist)}个RPM文件") + + # 关闭连接 + cursor.close() + conn.close() + + return rpmlist + + except Exception as e: + logger.error(f"从SQLite获取RPM列表失败: {str(e)}") + return [] + + def download_repodata_xml(self) -> Optional[str]: + """ + 下载repodata.xml文件 + + Returns: + 下载的文件路径，失败返回None + """ + # 首先尝试获取repomd.xml + repomd_url = HttpClient.join_url(self.base_url, "repodata/repomd.xml") + logger.info(f"尝试下载repomd.xml: {repomd_url}") + + # 生成缓存文件名 + cache_key = self.base_url.replace('://', '_').replace('/', '_').replace(':', '_') + repomd_path = os.path.join(self.cache_dir, f"{cache_key}_repomd.xml") + + # 下载repomd.xml + if not self.http_client.download_file(repomd_url, repomd_path): + logger.error(f"下载repomd.xml失败: {repomd_url}") + return None + + # 解析repomd.xml以找到primary.xml的位置 + try: + tree = ET.parse(repomd_path) + root = tree.getroot() + + # 查找primary.xml的位置 + primary_location = None + for data_element in root.findall('.//{http://linux.duke.edu/metadata/repo}data'): + if data_element.get('type') == 'primary': + location_element = data_element.find('.//{http://linux.duke.edu/metadata/repo}location') + if location_element is not None: + primary_location = location_element.get('href') + break + + if not primary_location: + logger.error("在repomd.xml中未找到primary.xml的位置") + return None + + # 下载primary.xml + primary_url = HttpClient.join_url(self.base_url, primary_location) + logger.info(f"找到primary.xml位置: {primary_url}") + + # 检查是否是压缩文件 + primary_path = os.path.join(self.cache_dir, f"{cache_key}_primary.xml") + compressed_path = None + + if primary_location.endswith('.gz'): + compressed_path = os.path.join(self.cache_dir, f"{cache_key}_primary.xml.gz") + if not self.http_client.download_file(primary_url, compressed_path): + logger.error(f"下载primary.xml.gz失败: {primary_url}") + return None + if not self.file_utils.extract_gz(compressed_path, primary_path): + logger.error(f"解压primary.xml.gz失败") + return None + elif primary_location.endswith('.bz2'): + compressed_path = os.path.join(self.cache_dir, f"{cache_key}_primary.xml.bz2") + if not self.http_client.download_file(primary_url, compressed_path): + logger.error(f"下载primary.xml.bz2失败: {primary_url}") + return None + if not self.file_utils.extract_bz2(compressed_path, primary_path): + logger.error(f"解压primary.xml.bz2失败") + return None + elif primary_location.endswith('.xz'): + compressed_path = os.path.join(self.cache_dir, f"{cache_key}_primary.xml.xz") + if not self.http_client.download_file(primary_url, compressed_path): + logger.error(f"下载primary.xml.xz失败: {primary_url}") + return None + if not self.file_utils.extract_xz(compressed_path, primary_path): + logger.error(f"解压primary.xml.xz失败") + return None + else: + # 直接下载未压缩的XML + if not self.http_client.download_file(primary_url, primary_path): + logger.error(f"下载primary.xml失败: {primary_url}") + return None + + return primary_path + + except Exception as e: + logger.error(f"处理repomd.xml时出错: {str(e)}") + return None + + def get_rpm_list_from_xml(self, xml_path: str) -> List[str]: + """ + 从primary.xml文件获取RPM列表 + + Args: + xml_path: primary.xml文件路径 + + Returns: + RPM列表 + """ + if not os.path.exists(xml_path): + logger.error(f"XML文件不存在: {xml_path}") + return [] + + rpmlist = [] + try: + # 解析XML文件 + tree = ET.parse(xml_path) + root = tree.getroot() + + # 查找所有包元素 + for pkg_element in root.findall('.//{http://linux.duke.edu/metadata/common}package'): + if pkg_element.get('type') != 'rpm': + continue + + # 获取包名 + name_element = pkg_element.find('.//{http://linux.duke.edu/metadata/common}name') + if name_element is None: + continue + name = name_element.text + + # 获取版本信息 + version_element = pkg_element.find('.//{http://linux.duke.edu/metadata/common}version') + if version_element is None: + continue + + epoch = version_element.get('epoch', '0') + version = version_element.get('ver', '') + release = version_element.get('rel', '') + + # 获取架构 + arch_element = pkg_element.find('.//{http://linux.duke.edu/metadata/common}arch') + if arch_element is None: + continue + arch = arch_element.text + + # 处理epoch + epoch_str = "" if not epoch or epoch == "0" else f"{epoch}:" + + # 生成RPM文件名 + rpm = f"{name}-{epoch_str}{version}-{release}.{arch}.rpm" + rpmlist.append(rpm) + + logger.info(f"从XML获取到{len(rpmlist)}个RPM文件") + return rpmlist + + except Exception as e: + logger.error(f"从XML获取RPM列表失败: {str(e)}") + return [] + + def compare_rpm_lists(self, list1: List[str], list2: List[str]) -> Dict[str, Any]: + """ + 比较两个RPM列表的差异 + + Args: + list1: 第一个RPM列表 + list2: 第二个RPM列表 + + Returns: + 比较结果 + """ + # 标准化RPM名称（去除版本信息，只保留包名） + def normalize_rpm_name(rpm: str) -> str: + # 移除.rpm后缀 + if rpm.lower().endswith('.rpm'): + rpm = rpm[:-4] + return rpm + + # 转换为集合以便比较 + set1 = set(list1) + set2 = set(list2) + + # 计算差异 + only_in_list1 = sorted(set1 - set2) + only_in_list2 = sorted(set2 - set1) + + # 检查是否完全相同 + is_identical = len(only_in_list1) == 0 and len(only_in_list2) == 0 + + # 构建结果 + result = { + "is_identical": is_identical, + "only_in_list1": only_in_list1, + "only_in_list2": only_in_list2, + "list1_count": len(list1), + "list2_count": len(list2), + "common_count": len(set1.intersection(set2)) + } + + return result + + def format_comparison_result(self, result: Dict[str, Any]) -> List[str]: + """ + 格式化比较结果 + + Args: + result: 比较结果字典 + + Returns: + 格式化后的文本行列表 + """ + output_lines = [] + + # 添加统计信息 + output_lines.append(f"列表1软件包数量: {result['list1_count']}") + output_lines.append(f"列表2软件包数量: {result['list2_count']}") + output_lines.append(f"共同软件包数量: {result.get('common_count', 0)}") + output_lines.append("") + + if result["is_identical"]: + output_lines.append("两个软件包列表完全相同") + output_lines.append("#####【测试通过】#####") + return output_lines + + # 报告第一个列表独有的项目 + if result["only_in_list1"]: + output_lines.append(f"仅在第一个列表中存在的软件包（{len(result['only_in_list1'])}个）：") + output_lines.extend(result["only_in_list1"]) + else: + output_lines.append("没有仅在第一个列表中存在的软件包。") + + output_lines.append("") + + # 报告第二个列表独有的项目 + if result["only_in_list2"]: + output_lines.append(f"仅在第二个列表中存在的软件包（{len(result['only_in_list2'])}个）：") + output_lines.extend(result["only_in_list2"]) + else: + output_lines.append("没有仅在第二个列表中存在的软件包。") + + return output_lines + + def get_all_rpm_sources(self) -> Dict[str, List[str]]: + """ + 获取所有可用的RPM列表来源 + + Returns: + 包含不同来源RPM列表的字典 + """ + results = {} + + # 1. 从HTML获取RPM列表 + html = self.get_html_content() + if html: + rpm_list_html = self.parse_rpm_list_from_html(html) + results["html"] = rpm_list_html + + # 2. 从SQLite获取RPM列表 + sqlite_path = self.download_and_extract_sqlite() + if sqlite_path: + rpm_list_sqlite = self.get_rpm_list_from_sqlite(sqlite_path) + results["sqlite"] = rpm_list_sqlite + + # 3. 从XML获取RPM列表 + xml_path = self.download_repodata_xml() + if xml_path: + rpm_list_xml = self.get_rpm_list_from_xml(xml_path) + results["xml"] = rpm_list_xml + + return results + + def compare_all_sources(self) -> Dict[str, Any]: + """ + 比较所有来源的RPM列表 + + Returns: + 比较结果 + """ + sources = self.get_all_rpm_sources() + + if not sources: + logger.error("没有找到任何RPM列表来源") + return {"error": "没有找到任何RPM列表来源"} + + results = {} + + # 如果有HTML和SQLite来源，比较它们 + if "html" in sources and "sqlite" in sources: + html_vs_sqlite = self.compare_rpm_lists(sources["html"], sources["sqlite"]) + results["html_vs_sqlite"] = html_vs_sqlite + + # 如果有HTML和XML来源，比较它们 + if "html" in sources and "xml" in sources: + html_vs_xml = self.compare_rpm_lists(sources["html"], sources["xml"]) + results["html_vs_xml"] = html_vs_xml + + # 如果有SQLite和XML来源，比较它们 + if "sqlite" in sources and "xml" in sources: + sqlite_vs_xml = self.compare_rpm_lists(sources["sqlite"], sources["xml"]) + results["sqlite_vs_xml"] = sqlite_vs_xml + + return results diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..549da20 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +# 项目依赖 +requests>=2.25.0 +beautifulsoup4>=4.9.3 +lxml>=4.6.3 +PyYAML>=5.4.1 diff --git a/script/01_repodata_check/README.md b/script/01_repodata_check/README.md new file mode 100644 index 0000000..fb654d9 --- /dev/null +++ b/script/01_repodata_check/README.md @@ -0,0 +1,27 @@ +# 测试项1 - repodata 检查 + +## 脚本简介 + +本脚本用于批量测试每周更新仓库的repodata是否正常。 + +## 使用方法 + +python3 repo_rpmlist_check_all.py + +## 输入 + +repo_configs1 + +1.每周仓库测试仓库地址 一般情况下不需要变更,可直接检查所有更新仓库； +2.可修改reo_config1中仓库为待其他测试仓库地址； +3.需注意仓库地址链接(eg:8U8)与其他仓库地址链接不一致； + +## 输出文件 + +完整日志：repo_rpmlist_check_results.log +失败记录：repo_rpmlist_check_fail.log + +## 脚本流程说明 + +脚本通过获取repodata内的数据库文件得到rpmlist 与网页解析获得rpmlist进行对比； +内网python3 环境，即可运行； diff --git a/script/01_repodata_check/repodata_rpmlist_check_all.py b/script/01_repodata_check/repodata_rpmlist_check_all.py new file mode 100644 index 0000000..c823eb3 --- /dev/null +++ b/script/01_repodata_check/repodata_rpmlist_check_all.py @@ -0,0 +1,354 @@ +# -*- coding:utf-8 -*- +# Time:2025/4/21 17:46 +# Author:GuoChao +# FileName:repo_rpmlist_check.py + +from bs4 import BeautifulSoup +import bz2 +import lzma +import shutil +import sqlite3 +import requests +from lxml import etree +import logging +logging.captureWarnings(True) +import gzip + +def Gethtml(baseurl): + path = f"{baseurl}/Packages/" + head = { + "User-Agent": "Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 80.0.3987.122 Safari / 537.36" + } + response = requests.get(path, headers=head, verify=False) + html = response.text + return html + +#使用bs4 进行网页解析，获取软件包列表 +def Getrpmlist(html): + rpmlist1=[] + rpmlist=[] + soup=BeautifulSoup(html,"html.parser") + for item in soup.find_all('a'): + rpmlist1.append(item['href']) + for i in rpmlist1: # 对url不能识别”+“进行修正 + if ".rpm" in i: + i = i.replace("%2B", "+") + rpmlist.append(i) + print('\n' +"通过html页面获取软件包： ",len(rpmlist)) + return rpmlist + + +def Getsqlite(baseurl): + """ + 从repodata中获取数据库压缩文件，下载并解压 + url 格式 例如 baseurl=https://update.cs2c.com.cn/NS/V10/8U6/os/adv/lic/AppStream/x86_64/os/ + """ + head = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" + } + + path = f"{baseurl}/repodata/" + + try: + response = requests.get(path, headers=head, verify=False, timeout=10) + if response.status_code != 200: + print(f"请求失败，状态码：{response.status_code}") + return None + + html = response.text + html = etree.HTML(html) + a_links = html.xpath("//a") + + for item in a_links: + item1 = item.get('href') + if item1.endswith("primary.sqlite.bz2"): + sqlite_url = '/'.join([path, item1]) + print(f"正在下载：{sqlite_url}") + response = requests.get(sqlite_url, headers=head, verify=False, timeout=10) + if response.status_code != 200: + print(f"下载失败，状态码：{response.status_code}") + return None + un_path = "repotest.sqlite.bz2" + with open(un_path, 'wb') as code: + code.write(response.content) + with bz2.BZ2File(un_path) as bz2file: + data = bz2file.read() + newfilepath = "repotest.sqlite" + with open(newfilepath, 'wb') as f: + f.write(data) + return newfilepath + + elif item1.endswith("primary.sqlite.xz"): + sqlite_url = '/'.join([path, item1]) + print(f"正在下载：{sqlite_url}") + response = requests.get(sqlite_url, headers=head, verify=False, timeout=100) + if response.status_code != 200: + print(f"下载失败，状态码：{response.status_code}") + return None + un_path = "repotest.sqlite.xz" + with open(un_path, 'wb') as code: + code.write(response.content) + with lzma.open(un_path, 'rb') as input: + with open("repotest.sqlite", 'wb') as output: + shutil.copyfileobj(input, output) + return "repotest.sqlite" + + elif item1.endswith("primary.sqlite.gz"): + sqlite_url = '/'.join([path, item1]) + print(f"正在下载：{sqlite_url}") + response = requests.get(sqlite_url, headers=head, verify=False, timeout=10) + if response.status_code != 200: + print(f"下载失败，状态码：{response.status_code}") + return None + un_path = "repotest.sqlite.gz" + with open(un_path, 'wb') as code: + code.write(response.content) + with gzip.open(un_path, 'rb') as input: + with open("repotest.sqlite", 'wb') as output: + shutil.copyfileobj(input, output) + return "repotest.sqlite" + + print("获取数据库文件失败，请检查！") + return None + + except Exception as e: + print(f"发生异常：{e}") + return None + + +def Getrpminfo(database): + """ + 连接sqlite数据库获取软件包信息 + """ + if database is None: + print("database 错误") + con = sqlite3.connect(database) + ##读取sqlite数据 + cursor = con.cursor() + ##创建游标cursor来执行executeＳＱＬ语句 + cursor.execute("SELECT * from packages") + content = cursor.execute("SELECT name, version, epoch, release, arch, rpm_sourcerpm, pkgId from packages") + rpmlist=[] + for row in cursor: + rpm=row[0] + "-" + row[1] + "-" + row[3] + "." + row[4] + ".rpm" + rpmlist.append(rpm) + print('\n' + '通过sqlite获取软件包：', len(rpmlist)) + return rpmlist + cursor.close() + con.close() + + +def compare_package_lists(package_list1, package_list2): + set1 = set(package_list1) + set2 = set(package_list2) + + only_in_list1 = set1 - set2 + only_in_list2 = set2 - set1 + + result = [] + if only_in_list1: + result.append("仅在第一个列表中存在的软件包：") + result.extend(sorted(only_in_list1)) + else: + result.append("没有仅在第一个列表中存在的软件包。") + + if only_in_list2: + result.append("仅在第二个列表中存在的软件包：") + result.extend(sorted(only_in_list2)) + else: + result.append("没有仅在第二个列表中存在的软件包。") + + if not only_in_list1 and not only_in_list2: + result.append('\n' +"两个软件包列表完全相同"+'\n'+"#####【测试通过】#####") + + for line in result: + print(line) + return result + + + +def save_comparison_result_to_file(result, file_path): + try: + with open(file_path, 'w', encoding='utf-8') as file: + for line in result: + file.write(line + '\n') + print(f"对比结果已成功保存到 {file_path}") + except Exception as e: + print(f"保存结果到文件时出现错误：{e}") + + + +if __name__ == "__main__": + + + repo_configs1 = [ + + { + "product": "V10-SP3-2403", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/V10SP3-2403/os/adv/lic/", + "architectures": ["x86_64", "aarch64", "loongarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-SP3", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/V10SP3/os/adv/lic/", + "architectures": ["x86_64", "aarch64", "loongarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-SP2", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/V10SP2/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-SP1.1", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/", + "architectures": ["x86_64", "aarch64", "mips64el", "loongarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-HPC", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/HPC/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-ZJ", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/V10-ZJ/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-2309A", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/2309A/os/adv/lic/", + "architectures": ["aarch64"], + "repo_types": ["updates"] + }, + { + "product": "V10-2309B", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/2309B/os/adv/lic/", + "architectures": ["aarch64"], + "repo_types": ["updates"] + }, + { + "product": "HostOS", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/HOST/SP3/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["updates"] + }, + { + "product": "Host-2309", + "base_url": "https://update.cs2c.com.cn/private_test/repo/HOST/2309/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["updates"] + }, + { + "product": "Host-2406", + "base_url": "https://update.cs2c.com.cn/private_test/repo/HOST/2406/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["updates"] + }, + { + "product": "V7.9", + "base_url": "https://update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["updates"] + }, + { + "product": "V6.10", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V6/V6.10/os/lic/", + "architectures": ["x86_64"], + "repo_types": ["updates"] + }, + { + "product": "8U2", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/8U2/os/adv/lic/", + "architectures": ["x86_64", "aarch64"], + "repo_types": ["BaseOS-updates", "AppStream-updates", "PowerTools-updates", "Plus-updates"] + }, + { + "product": "8U8", + "base_url": "https://update.cs2c.com.cn/private_test/repo/V10/8U8/os/adv/lic/", + "architectures": ["x86_64"], + "repo_types": ["BaseOS-updates", "AppStream-updates", "BaseOS", "AppStream","/kernel418","/kernel419"] + } + ] + + + if __name__ == "__main__": + # 初始化结果记录文件 + overall_result_file = "./repo_rpmlist_check_results.log" + fail_log_file = "./repo_rpmlist_check_fail.log" + + # 清空或创建结果文件 + with open(overall_result_file, 'w', encoding='utf-8') as f: + f.write("=== 整体测试结果 ===\n\n") + with open(fail_log_file, 'w', encoding='utf-8') as f: + f.write("=== 失败记录 ===\n\n") + + for config in repo_configs1: + product = config["product"] + base_url = config["base_url"] + + for arch in config["architectures"]: + for repo_type in config["repo_types"]: + # Construct full repository URL + #repo_url = f"{base_url}{repo_type}/{arch}/" # 通用 + # repo_url = f"{base_url}{repo_type}/{arch}/os/" # ns8.8 使用 + if "8U8" not in product: + repo_url = f"{base_url}{repo_type}/{arch}/" + else: + repo_url = f"{base_url}{repo_type}/{arch}/os/" + # Generate output filename + output_file = f"./rpm_check_{product}_{arch}_{repo_type}.txt" + + print(f"\n#####开始测试: {product} {arch} {repo_type}#####") + print(f"测试仓库地址: {repo_url}") + + try: + # Get RPM lists and compare + rpmlist1 = Getrpmlist(Gethtml(repo_url)) + rpmlist2 = Getrpminfo(Getsqlite(repo_url)) + result = compare_package_lists(rpmlist1, rpmlist2) + # Save full results to individual file + # save_comparison_result_to_file(result, output_file) + # 记录到整体结果文件 + with open(overall_result_file, 'a', encoding='utf-8') as f: + f.write(f"\n=== {product} {arch} {repo_type} ===\n") + f.write(f"Repository URL: {repo_url}\n") + for line in result: + f.write(line + '\n') + f.write("\n") + + # 检查是否有失败，如果有则记录到fail.log + if "测试通过" not in '\n'.join(result): + with open(fail_log_file, 'a', encoding='utf-8') as f: + f.write(f"\n=== {product} {arch} {repo_type} ===\n") + f.write(f"Repository URL: {repo_url}\n") + for line in result: + if "测试通过" not in line: + f.write(line + '\n') + f.write("\n") + + except Exception as e: + error_msg = f"Error processing {product} {arch} {repo_type}: {str(e)}" + print(error_msg) + + # 记录错误到整体结果文件和fail.log + with open(overall_result_file, 'a', encoding='utf-8') as f: + f.write(f"\n=== {product} {arch} {repo_type} ===\n") + f.write(f"Repository URL: {repo_url}\n") + f.write(f"ERROR: {error_msg}\n\n") + + with open(fail_log_file, 'a', encoding='utf-8') as f: + f.write(f"\n=== {product} {arch} {repo_type} ===\n") + f.write(f"Repository URL: {repo_url}\n") + f.write(f"ERROR: {error_msg}\n\n") + + # 仍然保存错误到单独文件 + with open(f"./error_{product}_{arch}_{repo_type}.log", "w") as f: + f.write(f"Error processing {repo_url}:\n{error_msg}") + + print(f"\n所有测试完成，整体结果已保存到 {overall_result_file}") + print(f"失败记录已保存到 {fail_log_file}") diff --git a/script/02_package_signatures/README.md b/script/02_package_signatures/README.md new file mode 100644 index 0000000..f37873f --- /dev/null +++ b/script/02_package_signatures/README.md @@ -0,0 +1,37 @@ + 测试项1 - 安装包签名检查 + +## 脚本简介 + +本脚本用于批量测试软件包签名的方式 + +## 使用方法 +# ./check_pgpsig.sh + +--- + +## 输入文件 +- `pkg`：每行一个 RPM 包名，。 +--- + +## 输出文件 + +- 没有签名的软件包名 +--- + +## 脚本流程说明 + +1. 使用 find 命令递归查找当前目录下的所有 RPM 文件 +2. 使用 rpm -qp 命令查询 RPM 包的头信息 +3. --qf 选项指定输出格式，包含包名 (%{NAME}) 和 GPG 签名信息 (%{RSAHEADER:pgpsig}) +4. 如果签名信息中包含 "none"，表示未签名，将结果同时输出到标准输出和 nogpg.list 文件。 +5. 否则，表示已签名，将结果追加到 gpg.list 文件。 + +--- + +## 注意事项 + +- 需要 root 权限运行。 +- 系统需启用 YUM 并配置好可用仓库。 +- 安装失败或卸载失败的错误日志会保存在 `Error` 字段中，方便后续排查。 + +--- diff --git a/script/02_package_signatures/check_pgpsig.sh b/script/02_package_signatures/check_pgpsig.sh new file mode 100644 index 0000000..1c45e82 --- /dev/null +++ b/script/02_package_signatures/check_pgpsig.sh @@ -0,0 +1,11 @@ +#!/bin/bash +LANG=C +for pkg in `find . -type f -name '*.rpm'` +do + RES=`rpm -qp --qf '%{NAME} %{RSAHEADER:pgpsig}\n' $pkg` + if echo $RES | grep 'none';then +echo $RES | tee -a nogpg.list + else +echo $RES >> gpg.list + fi +done \ No newline at end of file diff --git a/script/03_package_hash/README.md b/script/03_package_hash/README.md new file mode 100644 index 0000000..7b4634e --- /dev/null +++ b/script/03_package_hash/README.md @@ -0,0 +1,38 @@ + 测试项1 - 安装包哈希值 + +## 脚本简介 + +这个 Bash 脚本的主要功能是递归查找当前目录下的所有 RPM 包文件，然后比较每个 RPM 包的本地 MD5 签名与 Koji 构建系统中记录的 MD5 签名是否一致 + +## 使用方法 +# ./check_pgpsig.sh + +--- + +## 输入文件 +- `pkg`：每行一个 RPM 包名，。 +-使用 find 命令递归查找当前目录下的所有 RPM 文件 +--- + +## 输出文件 +-：MD5 值匹配的和不匹配的，并分别将结果输出到 equal.txt 和 noquual.txt 文件中 + +--- + +## 脚本流程说明 + +1. 使用 find 命令递归查找当前目录下的所有 RPM 文件 +2. 使用 rpm -qp 命令查询 RPM 包的头信息 +3. --qf 选项指定输出格式，包含包名 (%{NAME}) 和 GPG 签名信息 (%{RSAHEADER:pgpsig}) +4. 如果签名信息中包含 "none"，表示未签名，将结果同时输出到标准输出和 nogpg.list 文件。 +5. 否则，表示已签名，将结果追加到 gpg.list 文件。 + +--- + +## 注意事项 + +- 需要 root 权限运行。 +- 系统需启用 YUM 并配置好可用仓库。 +- 安装失败或卸载失败的错误日志会保存在 `Error` 字段中，方便后续排查。 + +--- diff --git a/script/03_package_hash/compare_file_hash.sh b/script/03_package_hash/compare_file_hash.sh new file mode 100644 index 0000000..50255d1 --- /dev/null +++ b/script/03_package_hash/compare_file_hash.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 定义要处理的文件夹路径 +for rpm_file in $(find . -type f -name '*.rpm'); do + # 去除 ./ 前缀 + clean_rpm_file="${rpm_file#./}" + + # 检查文件是否存在 + if [ -f "$clean_rpm_file" ]; then + # 执行 rpm 命令获取 MD5 值 + rpm_md5=$(rpm -qp --queryformat '%{SIGMD5}\n' "$clean_rpm_file" 2>&1 | awk '/^[0-9a-fA-F]{32}$/ {print $0}') + + echo $rpm_md5 + # 执行 koji 命令获取 MD5 值 + koji_md5=$(koji rpminfo "$clean_rpm_file" | grep 'SIGMD5:' | awk '{print $2}') + echo $koji_md5 + + # 比较两个 MD5 值 + if [ "$rpm_md5" = "$koji_md5" ]; then + echo "File: $clean_rpm_file" >> equal.txt + echo "rpm MD5: $rpm_md5" >> equal.txt + echo "koji MD5: $koji_md5" >> equal.txt + echo "------------------------" >> equal.txt + else + echo "File: $clean_rpm_file" >> noquual.txt + echo "rpm MD5: $rpm_md5" >> noquual.txt + echo "koji MD5: $koji_md5" >> noquual.txt + echo "------------------------" >> noquual.txt + fi + fi +done \ No newline at end of file diff --git a/script/04_package_tps_test/README.md b/script/04_package_tps_test/README.md new file mode 100644 index 0000000..ab73039 --- /dev/null +++ b/script/04_package_tps_test/README.md @@ -0,0 +1,62 @@ +# 测试项2 - 安装卸载测试脚本说明 + +## 脚本简介 + +本脚本用于批量测试指定架构和产品下的软件包是否能被 **成功安装** 并 **成功卸载**。 +读取 `Pkglist` 中的包名，分别进行安装和卸载，并记录详细日志与结果。 + +## 使用方法 +# sh test.sh +示例： +# sh test.sh V10 x86_64 +--- + +## 输入文件 +- `Pkglist`：文本文件，每行一个 RPM 包名，作为测试目标（不包含 `.rpm` 后缀）。 +--- + +## 输出文件 + +- `测试项2-__install_results.csv` + 安装卸载测试结果表格，包含四列： + - `Package`：软件包名 + - `Install Status`：安装是否成功（成功/失败） + - `Remove Status`：卸载是否成功（成功/失败） + - `Error`：失败时的错误信息 + +- `测试项2-__install_test.log` + 完整安装卸载过程日志。 +--- + +## 脚本流程说明 + +1. 读取参数 `` 与 ``。 +2. 逐行读取 `Pkglist` 中的软件包名。 +3. 对每个包执行： + - `yum install -y `（记录安装状态） + - 安装成功后再执行 `yum remove -y `（记录卸载状态） +4. 写入最终 CSV 与日志文件。 + +--- + +## 注意事项 + +- 需要 root 权限运行。 +- 系统需启用 YUM 并配置好可用仓库。 +- 安装失败或卸载失败的错误日志会保存在 `Error` 字段中，方便后续排查。 + +--- +compare-0522.py + +## 脚本简介 + +本脚本生成仓库测试的多个脚本,安装本周更新的的软件包以及多版本升降级脚本,把每个产品都生成一个文件夹供测试使用 +## 使用方法 +# python3 compare-0522.py +生成文件内容： +install_remove_test.sh （安装测试脚本） +Pkglist-V10SP4-x86 （本周仓库更新的二进制包列表） +V10SP3-2403-x86分析报告-分析报告-20250527-1406.xlsx （仓库分析报告） +test_upgrade_downgrade_V10SP3-2403-x86--20250527-1406.sh （本周更新包升降级测试脚本） + + diff --git a/script/04_package_tps_test/README_compare_ldy.md b/script/04_package_tps_test/README_compare_ldy.md new file mode 100644 index 0000000..78a82db --- /dev/null +++ b/script/04_package_tps_test/README_compare_ldy.md @@ -0,0 +1,91 @@ +# compare.py 使用说明文档 + +## 脚本简介 + +该脚本用于比较测试仓库与发布仓库新增/减少的软件包的构建差异: + +- 自动下载并解压 `primary.sqlite` 数据库文件 +- 提取 SRPM 与 Binary NVR 信息 +- 分析多个仓库中同一 SRPM 的二进制构建差异 +- 输出本周重点分析的 SRPM 差异（新增/减少） +- 输出多仓库整体 SRPM 构建分布情况到 Excel 文件 +--- + +## 使用方法 + +python3 compare.py + +脚本默认读取当前目录下的 `config.ini` 配置文件。 +--- + +## 配置说明 + +`config.ini` 配置格式如下： + +[V10sp4-x86] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/x86_64/ + updates=https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/updates/x86_64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/x86_64/ +filename = V10SP3-2403-x86分析报告 + +[V10sp3-x86] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/x86_64/ + updates=https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/updates/x86_64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/V10SP3/os/adv/lic/updates/x86_64/ +filename = V10SP3-x86分析报告 + +支持多个产品段落，每个段落会分别输出一个分析结果。 + +--- + +## 输出内容说明 + +脚本会输出一个 `.xlsx` 文件，包含以下 Sheet： + +1. `仓库所有SRPM分析`：列出所有仓库中 SRPM → Binary 包映射与差异信息 +2. `外网仓库本周重点分析的SRPM`：筛选重点 SRPM 在 base 与 updates 仓库中的构建差异 +3. `测试仓库本周重点分析的SRPM`：筛选重点 SRPM 在 base 与 updates_test 仓库中的构建差异 +4. `测试仓库对比`：展示 updates 与 updates_test 仓库中新增/减少的 SRPM 与 Binary 包 +5. `仓库原始数据`（可选）：包含各仓库原始包信息 +--- + +## 分析字段说明 + +输出的主要字段如下： + +- `Repo`：仓库名 +- `SRPM Name`：源码包名（无版本） +- `SRPM Version`：源码包版本号（version-release） +- `Binary Package List`：构建出的二进制包名（不含版本） +- `Binary NVR List`：完整二进制包 NVR +- `Binary Count`：该 SRPM 构建的二进制包数量 +- `Binary Count Changed`：是否存在包数量变动（Yes/No） +- `Added Binaries` / `Removed Binaries`：相对于其他版本新增/减少的二进制包名 + +--- + +## 依赖组件 + +- Python 3.x +- `requests` +- `lxml` +- `pandas` +- `openpyxl` + +安装依赖示例： + +```bash +pip install requests lxml pandas openpyxl +``` +--- + +## 示例输出文件 + +输出文件名格式如下： + +``` +-差异分析-YYYYMMDD-HHMM.xlsx +``` + diff --git a/script/04_package_tps_test/compare-0522.py b/script/04_package_tps_test/compare-0522.py new file mode 100644 index 0000000..5fa6799 --- /dev/null +++ b/script/04_package_tps_test/compare-0522.py @@ -0,0 +1,371 @@ +from collections import defaultdict +import bz2 +import lzma +import shutil +import sqlite3 +import requests +from lxml import etree +import configparser +import datetime +import pandas as pd +import openpyxl +import gzip +import os +from openpyxl.styles import Font, PatternFill +import urllib3 +from test_srpm_up_down import gen_updown_test +#from generate_test_summary import generate_test_summary_excel + + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +# --------------------- +# 下载 primary.sqlite 并解压 +# --------------------- +def Getsqlite_primary(baseurl): + head = { + "User-Agent": "Mozilla/5.0" + } + path = f"{baseurl}repodata/" + try: + response = requests.get(path, headers=head, verify=False) + html = response.text + html = etree.HTML(html) + a_links = html.xpath("//a") + + for item in a_links: + item1 = item.get('href') + if item1.endswith("primary.sqlite.bz2"): + sqlite_url = '/'.join([path, item1]) + print(sqlite_url) + response = requests.get(sqlite_url, headers=head, verify=False) + un_path = "repotest.sqlite.bz2" + with open(un_path, 'wb') as code: + code.write(response.content) + bz2file = bz2.BZ2File(un_path) + data = bz2file.read() + newfilepath = "repotest.sqlite" + open(newfilepath, 'wb').write(data) + return newfilepath + + elif item1.endswith("primary.sqlite.xz"): + sqlite_url = '/'.join([path, item1]) + print(sqlite_url) + response = requests.get(sqlite_url, headers=head, verify=False) + un_path = "repotest.sqlite.xz" + with open(un_path, 'wb') as code: + code.write(response.content) + with lzma.open(un_path, 'rb') as input: + with open("repotest.sqlite", 'wb') as output: + shutil.copyfileobj(input, output) + return "repotest.sqlite" + + elif item1.endswith("primary.sqlite.gz"): + sqlite_url = '/'.join([path, item1]) + response = requests.get(sqlite_url, headers=head, verify=False) + un_path = "repotest.sqlite.gz" + with open(un_path, 'wb') as code: + code.write(response.content) + with gzip.open(un_path, 'rb') as input: + with open("repotest.sqlite", 'wb') as output: + shutil.copyfileobj(input, output) + return "repotest.sqlite" + + print("获取数据库文件失败，请检查！") + return None + except Exception as e: + print("发生异常：", e) + return None + +# --------------------- +# 获取包信息 +# --------------------- +def get_package_info_from_db1(baseurl): + db_path = Getsqlite_primary(baseurl) + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute("SELECT pkgkey, name, arch, version, release, rpm_sourcerpm FROM packages") + result = cursor.fetchall() + conn.close() + pkg_info = {} + for pkgkey, name, arch, version, release, rpm_sourcerpm in result: + pkg_info[pkgkey] = { + "pkgkey": pkgkey, + "name": name, + "arch": arch, + "version": version, + "release": release, + "source": rpm_sourcerpm, + } + return pkg_info + +def extract_srpm_nvr(pkg_info): + return set(info['source'].replace('.src.rpm', '').replace('.nosrc.rpm', '') for info in pkg_info.values() if info['source']) + +def extract_binary_nvr(pkg_info): + return set(f"{info['name']}-{info['version']}-{info['release']}.{info['arch']}" for info in pkg_info.values()) + +def extract_srpm_names_from_nvr_list(nvr_list): + srpm_names = set() + for nvr in nvr_list: + parts = nvr.rsplit('-', 2) + if len(parts) == 3: + srpm_names.add(parts[0]) + return sorted(srpm_names) + +def analyze_binary_distribution_all_repos(all_pkg_info_by_repo): + srpm_versions = defaultdict(lambda: defaultdict(lambda: defaultdict(set))) + for repo_name, pkg_info in all_pkg_info_by_repo.items(): + for info in pkg_info.values(): + source_rpm = info['source'] + binary_name = info['name'] + try: + srpm_base = source_rpm.replace('.src.rpm', '').replace('.nosrc.rpm', '') + srpm_name, srpm_version, srpm_release = srpm_base.rsplit('-', 2) + srpm_fullver = f"{srpm_version}-{srpm_release}" + except Exception: + continue + srpm_versions[srpm_name][repo_name][srpm_fullver].add(binary_name) + + srpm_change_flags = {} + srpm_all_binaries = {} + for srpm_name, repo_versions in srpm_versions.items(): + all_binary_counts = [] + all_binaries = set() + for version_map in repo_versions.values(): + for binaries in version_map.values(): + all_binary_counts.append(len(binaries)) + all_binaries.update(binaries) + srpm_change_flags[srpm_name] = len(set(all_binary_counts)) > 1 + srpm_all_binaries[srpm_name] = all_binaries + + records = [] + for srpm_name, repo_versions in srpm_versions.items(): + changed = srpm_change_flags[srpm_name] + all_binaries = srpm_all_binaries[srpm_name] + for repo_name, versions in repo_versions.items(): + pkg_info_map = all_pkg_info_by_repo[repo_name] + for version, binaries in versions.items(): + added = sorted(binaries - all_binaries) + removed = sorted(all_binaries - binaries) + binary_nvrs = sorted([ + f"{info['name']}-{info['version']}-{info['release']}.{info['arch']}" + for info in pkg_info_map.values() + if info["name"] in binaries and + info["source"].replace('.src.rpm', '').replace('.nosrc.rpm', '').endswith(version) + ]) + records.append({ + "Repo": repo_name, + "SRPM Name": srpm_name, + "SRPM Version": version, + "Binary Count": len(binaries), + "Binary Package List": ' '.join(sorted(binaries)), + "Binary NVR List": ' '.join(binary_nvrs), + "Binary Count Changed": "Yes" if changed else "No", + "Added Binaries": ' '.join(added), + "Removed Binaries": ' '.join(removed), + }) + df = pd.DataFrame(records) + df.sort_values(by=["SRPM Name", "SRPM Version", "Repo"], inplace=True) + return df + +def write_test_comparison_sheet(wb, added_srpms, removed_srpms, added_bins, removed_bins): + sheet = wb.create_sheet("测试仓库对比") + sheet.append(["新增 SRPM（仅测试仓库对比）", "新增 Binary NVR（仅测试仓库对比）"]) + for i in range(max(len(added_srpms), len(added_bins))): + sheet.append([added_srpms[i] if i < len(added_srpms) else "", added_bins[i] if i < len(added_bins) else ""]) + sheet.append([]) + sheet.append(["减少 SRPM（仅测试仓库对比）", "减少 Binary NVR（仅测试仓库对比）"]) + for i in range(max(len(removed_srpms), len(removed_bins))): + sheet.append([removed_srpms[i] if i < len(removed_srpms) else "", removed_bins[i] if i < len(removed_bins) else ""]) + +def write_filtered_analysis_sheet(analysis_df, target_srpms, worksheet): + filtered_df = analysis_df[analysis_df["SRPM Name"].isin(target_srpms)] + worksheet.append(list(filtered_df.columns)) + for _, row in filtered_df.iterrows(): + worksheet.append(list(row.values)) + +def write_analysis_to_excel(analysis_df, repo_name, worksheet, is_first=False): + if is_first: + worksheet.append(["Repo"] + list(analysis_df.columns)) + for _, row in analysis_df.iterrows(): + worksheet.append([repo_name] + list(row.values)) + +def load_product_config(): + config = configparser.ConfigParser() + config.read('config.ini', encoding='utf-8') + all_products = {} + for section in config.sections(): + if not config.has_option(section, 'baseurls'): + continue + baseurls_raw = config.get(section, 'baseurls', fallback='') + baseurl_lines = [line.strip() for line in baseurls_raw.splitlines() if line.strip()] + baseurl_dict = {} + for line in baseurl_lines: + if '=' in line: + name, url = line.split('=', 1) + baseurl_dict[name.strip()] = url.strip() + filename = config.get(section, 'filename', fallback=section) + updates_test = config.get(section, 'updates_test', fallback=None) + all_products[section] = { + "baseurls": baseurl_dict, + "filename": filename, + "updates_test": updates_test + } + return all_products + +def compare_updates_and_test(updates_url, test_url): + new_info = get_package_info_from_db1(updates_url) + test_info = get_package_info_from_db1(test_url) + new_srpms = extract_srpm_nvr(new_info) + test_srpms = extract_srpm_nvr(test_info) + new_bins = extract_binary_nvr(new_info) + test_bins = extract_binary_nvr(test_info) + added_srpms = sorted(test_srpms - new_srpms) + removed_srpms = sorted(new_srpms - test_srpms) + added_bins = sorted(test_bins - new_bins) + removed_bins = sorted(new_bins - test_bins) + focus_srpms = extract_srpm_names_from_nvr_list(added_srpms + removed_srpms) + return added_srpms, removed_srpms, added_bins, removed_bins, focus_srpms + +def run_for_product(product_name, baseurl_map, filename_prefix, updates_test_url, show_raw=False): + print(f"\n🔍 正在分析产品：{product_name}") + + output_dir = os.path.join("output", product_name) + os.makedirs(output_dir, exist_ok=True) + timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M") + excel_path = os.path.join(output_dir, f"{filename_prefix}-分析报告-{timestamp}.xlsx") + pkglist_path = os.path.join(output_dir, f"Pkglist-{product_name}") + + wb = openpyxl.Workbook() + if show_raw: + raw_sheet = wb.create_sheet("仓库原始数据") + analysis_sheet = wb.create_sheet("仓库所有SRPM分析") + focus_sheet_external = wb.create_sheet("外网仓库本周重点分析的SRPM") + focus_sheet_test = wb.create_sheet("测试仓库本周重点分析的SRPM") + wb.remove(wb.active) + + all_pkg_info_by_repo = {} + if show_raw: + is_first_raw = True + for repo_name, baseurl in baseurl_map.items(): + info = get_package_info_from_db1(baseurl) + all_pkg_info_by_repo[repo_name] = info + write_Excel(info, repo_name, raw_sheet, is_first=is_first_raw) + is_first_raw = False + else: + for repo_name, baseurl in baseurl_map.items(): + all_pkg_info_by_repo[repo_name] = get_package_info_from_db1(baseurl) + + analysis_df = analyze_binary_distribution_all_repos(all_pkg_info_by_repo) + write_analysis_to_excel(analysis_df, "Merged", analysis_sheet, is_first=True) + + updates_url = baseurl_map.get("updates") + base_url = baseurl_map.get("base") + if updates_url and updates_test_url: + added_srpms, removed_srpms, added_bins, removed_bins, focus_srpms = compare_updates_and_test(updates_url, updates_test_url) + write_test_comparison_sheet(wb, added_srpms, removed_srpms, added_bins, removed_bins) + with open(pkglist_path, 'w') as f: + for nvr in added_bins: + f.write(nvr + '\n') + + ext_info = { + 'base': get_package_info_from_db1(base_url), + 'updates': get_package_info_from_db1(updates_url) + } + ext_df = analyze_binary_distribution_all_repos(ext_info) + write_filtered_analysis_sheet(ext_df, focus_srpms, focus_sheet_external) + + test_info = { + 'base': get_package_info_from_db1(base_url), + 'updates_test': get_package_info_from_db1(updates_test_url) + } + test_df = analyze_binary_distribution_all_repos(test_info) + write_filtered_analysis_sheet(test_df, focus_srpms, focus_sheet_test) + + wb.save(excel_path) + print(f"✅ {product_name} 分析完成，输出文件：{excel_path}") + + # ✅ 生成安装卸载测试脚本（不需要传参） + install_test_script_path = os.path.join(output_dir, "install_remove_test.sh") + with open(install_test_script_path, "w") as f: + f.write(f"""#!/bin/bash + +PACKAGE_LIST=Pkglist-{product_name} +OUTPUT_FILE={product_name}_install_results.csv +LOG_FILE={product_name}_install_test.log + +echo \"Package,Install Status,Remove Status,Error\" > "$OUTPUT_FILE" +echo \"===== 安装测试日志开始 =====\" > "$LOG_FILE" + +while IFS= read -r package +do + if [ -z "$package" ]; then + continue + fi + + echo "正在安装: $package" | tee -a "$LOG_FILE" + install_output=$(yum --setopt=timeout=300 --setopt=retries=10 install -y "$package" 2>&1) + install_status=$? + + if [ $install_status -eq 0 ]; then + install_result=成功 + else + install_result=失败 + fi + + echo "$install_output" >> "$LOG_FILE" + echo "正在卸载: $package" | tee -a "$LOG_FILE" + remove_output=$(yum remove -y "$package" 2>&1) + remove_status=$? + + if [ $remove_status -eq 0 ]; then + remove_result=成功 + else + remove_result=失败 + fi + + echo "$remove_output" >> "$LOG_FILE" + + if [ "$install_result" = "失败" ]; then + error_msg="$install_output" + elif [ "$remove_result" = "失败" ]; then + error_msg="$remove_output" + else + error_msg="" + fi + + echo "$package,$install_result,$remove_result,\"$error_msg\"" >> "$OUTPUT_FILE" + echo "--------------------------------" >> "$LOG_FILE" +done < "$PACKAGE_LIST" + +echo \"===== 安装测试日志结束 =====\" >> "$LOG_FILE" +echo \"测试完成，结果保存到 $OUTPUT_FILE，过程日志保存到 $LOG_FILE\" +""") + os.chmod(install_test_script_path, 0o755) + + # 🔧 生成 SRPM 升降级测试脚本 + success = gen_updown_test([excel_path], filter_choice="All", sheet_name="测试仓库本周重点分析的SRPM") + if success: + for fname in os.listdir("."): + if fname.startswith("test_upgrade_downgrade_") and fname.endswith(".sh"): + shutil.move(fname, os.path.join(output_dir, fname)) + elif fname.startswith("test_results_") or fname.startswith("test_process_"): + shutil.move(fname, os.path.join(output_dir, fname)) + # # ✅ 汇总测试结果（安装卸载 + 升级降级） + # generate_test_summary_excel(output_dir, product_name) + + +def main(): + products = load_product_config() + for name, conf in products.items(): + run_for_product( + product_name=name, + baseurl_map=conf['baseurls'], + filename_prefix=conf['filename'], + updates_test_url=conf.get('updates_test'), + show_raw=False + ) + +if __name__ == '__main__': + main() diff --git a/script/04_package_tps_test/compare.py b/script/04_package_tps_test/compare.py new file mode 100644 index 0000000..89b33d7 --- /dev/null +++ b/script/04_package_tps_test/compare.py @@ -0,0 +1,319 @@ +from collections import defaultdict +import bz2 +import lzma +import shutil +import sqlite3 +import requests +from lxml import etree +import configparser +import datetime +import pandas as pd +import openpyxl +import gzip +from openpyxl.styles import Font, PatternFill +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +# --------------------- +# 下载 primary.sqlite 并解压 +# --------------------- +def Getsqlite_primary(baseurl): + """ + 从repodata中获取数据库压缩文件，下载并解压 + url 格式 例如 basrutl=https://update.cs2c.com.cn/NS/V10/8U6/os/adv/lic/AppStream/x86_64/os/ + """ + head = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" + } + path = f"{baseurl}repodata/" + try: + response = requests.get(path, headers=head, verify=False) + html = response.text + html = etree.HTML(html) + a_links = html.xpath("//a") + + for item in a_links: + item1 = item.get('href') + if item1.endswith("primary.sqlite.bz2"): + sqlite_url = '/'.join([path, item1]) + print(sqlite_url) + response = requests.get(sqlite_url, headers=head, verify=False) + un_path = "repotest.sqlite.bz2" + with open(un_path, 'wb') as code: + code.write(response.content) + bz2file = bz2.BZ2File(un_path) + data = bz2file.read() + newfilepath = "repotest.sqlite" + open(newfilepath, 'wb').write(data) + return newfilepath + + elif item1.endswith("primary.sqlite.xz"): + sqlite_url = '/'.join([path, item1]) + response = requests.get(sqlite_url, headers=head, verify=False) + un_path = "repotest.sqlite.xz" + + with open(un_path, 'wb') as code: + code.write(response.content) + + with lzma.open(un_path, 'rb') as input: + with open("repotest.sqlite", 'wb') as output: + shutil.copyfileobj(input, output) + return "repotest.sqlite" + + elif item1.endswith("primary.sqlite.gz"): + sqlite_url = '/'.join([path, item1]) + print(sqlite_url) + response = requests.get(sqlite_url, headers=head, verify=False) + un_path = "repotest.sqlite.gz" + + with open(un_path, 'wb') as code: + code.write(response.content) + + with gzip.open(un_path, 'rb') as input: + with open("repotest.sqlite", 'wb') as output: + shutil.copyfileobj(input, output) + return "repotest.sqlite" + + print("获取数据库文件失败，请检查！") + return None + + except Exception as e: + print("发生异常：", e) + return None + + +# --------------------- +# 获取包信息 +# --------------------- +def get_package_info_from_db1(baseurl): + db_path = Getsqlite_primary(baseurl) + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute("SELECT pkgkey, name, arch, version, release, rpm_sourcerpm FROM packages") + result = cursor.fetchall() + conn.close() + pkg_info = {} + for pkgkey, name, arch, version, release, rpm_sourcerpm in result: + pkg_info[pkgkey] = { + "pkgkey": pkgkey, + "name": name, + "arch": arch, + "version": version, + "release": release, + "source": rpm_sourcerpm, + } + return pkg_info + +# --------------------- +# 提取 SRPM 名称/NVR +# --------------------- +def extract_srpm_nvr(pkg_info): + return set(info['source'].replace('.src.rpm', '').replace('.nosrc.rpm', '') for info in pkg_info.values() if info['source']) + +def extract_binary_nvr(pkg_info): + return set(f"{info['name']}-{info['version']}-{info['release']}.{info['arch']}" for info in pkg_info.values()) + +def extract_srpm_names_from_nvr_list(nvr_list): + srpm_names = set() + for nvr in nvr_list: + parts = nvr.rsplit('-', 2) + if len(parts) == 3: + srpm_names.add(parts[0]) + return sorted(srpm_names) + +# --------------------- +# 差异分析（多仓库） +# --------------------- +def analyze_binary_distribution_all_repos(all_pkg_info_by_repo): + srpm_versions = defaultdict(lambda: defaultdict(lambda: defaultdict(set))) + for repo_name, pkg_info in all_pkg_info_by_repo.items(): + for info in pkg_info.values(): + source_rpm = info['source'] + binary_name = info['name'] + try: + srpm_base = source_rpm.replace('.src.rpm', '').replace('.nosrc.rpm', '') + srpm_name, srpm_version, srpm_release = srpm_base.rsplit('-', 2) + srpm_fullver = f"{srpm_version}-{srpm_release}" + except Exception: + continue + srpm_versions[srpm_name][repo_name][srpm_fullver].add(binary_name) + + srpm_change_flags = {} + srpm_all_binaries = {} + for srpm_name, repo_versions in srpm_versions.items(): + all_binary_counts = [] + all_binaries = set() + for version_map in repo_versions.values(): + for binaries in version_map.values(): + all_binary_counts.append(len(binaries)) + all_binaries.update(binaries) + srpm_change_flags[srpm_name] = len(set(all_binary_counts)) > 1 + srpm_all_binaries[srpm_name] = all_binaries + + records = [] + for srpm_name, repo_versions in srpm_versions.items(): + changed = srpm_change_flags[srpm_name] + all_binaries = srpm_all_binaries[srpm_name] + for repo_name, versions in repo_versions.items(): + pkg_info_map = all_pkg_info_by_repo[repo_name] + for version, binaries in versions.items(): + added = sorted(binaries - all_binaries) + removed = sorted(all_binaries - binaries) + binary_nvrs = sorted([ + f"{info['name']}-{info['version']}-{info['release']}.{info['arch']}" + for info in pkg_info_map.values() + if info["name"] in binaries and + info["source"].replace('.src.rpm', '').replace('.nosrc.rpm', '').endswith(version) + ]) + records.append({ + "Repo": repo_name, + "SRPM Name": srpm_name, + "SRPM Version": version, + "Binary Count": len(binaries), + "Binary Package List": ' '.join(sorted(binaries)), + "Binary NVR List": ' '.join(binary_nvrs), + "Binary Count Changed": "Yes" if changed else "No", + "Added Binaries": ' '.join(added), + "Removed Binaries": ' '.join(removed), + }) + df = pd.DataFrame(records) + df.sort_values(by=["SRPM Name", "SRPM Version", "Repo"], inplace=True) + return df + +# --------------------- +# 对比测试仓库并返回重点 SRPM +# --------------------- +def compare_updates_and_test(updates_url, test_url): + new_info = get_package_info_from_db1(updates_url) + test_info = get_package_info_from_db1(test_url) + new_srpms = extract_srpm_nvr(new_info) + test_srpms = extract_srpm_nvr(test_info) + new_bins = extract_binary_nvr(new_info) + test_bins = extract_binary_nvr(test_info) + added_srpms = sorted(test_srpms - new_srpms) + removed_srpms = sorted(new_srpms - test_srpms) + added_bins = sorted(test_bins - new_bins) + removed_bins = sorted(new_bins - test_bins) + focus_srpms = extract_srpm_names_from_nvr_list(added_srpms + removed_srpms) + return added_srpms, removed_srpms, added_bins, removed_bins, focus_srpms + +def write_test_comparison_sheet(wb, added_srpms, removed_srpms, added_bins, removed_bins): + sheet = wb.create_sheet("测试仓库对比") + sheet.append(["新增 SRPM（仅测试仓库对比）", "新增 Binary NVR（仅测试仓库对比）"]) + for i in range(max(len(added_srpms), len(added_bins))): + sheet.append([added_srpms[i] if i < len(added_srpms) else "", added_bins[i] if i < len(added_bins) else ""]) + sheet.append([]) + sheet.append(["减少 SRPM（仅测试仓库对比）", "减少 Binary NVR（仅测试仓库对比）"]) + for i in range(max(len(removed_srpms), len(removed_bins))): + sheet.append([removed_srpms[i] if i < len(removed_srpms) else "", removed_bins[i] if i < len(removed_bins) else ""]) + +# --------------------- +# 写入分析表 +# --------------------- +def write_filtered_analysis_sheet(analysis_df, target_srpms, worksheet): + filtered_df = analysis_df[analysis_df["SRPM Name"].isin(target_srpms)] + worksheet.append(list(filtered_df.columns)) + for _, row in filtered_df.iterrows(): + worksheet.append(list(row.values)) + +def write_analysis_to_excel(analysis_df, repo_name, worksheet, is_first=False): + if is_first: + worksheet.append(["Repo"] + list(analysis_df.columns)) + for _, row in analysis_df.iterrows(): + worksheet.append([repo_name] + list(row.values)) + +# --------------------- +# 配置读取 +# --------------------- +def load_product_config(): + config = configparser.ConfigParser() + config.read('config.ini', encoding='utf-8') + all_products = {} + for section in config.sections(): + if not config.has_option(section, 'baseurls'): + continue + baseurls_raw = config.get(section, 'baseurls', fallback='') + baseurl_lines = [line.strip() for line in baseurls_raw.splitlines() if line.strip()] + baseurl_dict = {} + for line in baseurl_lines: + if '=' in line: + name, url = line.split('=', 1) + baseurl_dict[name.strip()] = url.strip() + filename = config.get(section, 'filename', fallback=section) + updates_test = config.get(section, 'updates_test', fallback=None) + all_products[section] = { + "baseurls": baseurl_dict, + "filename": filename, + "updates_test": updates_test + } + return all_products + +# --------------------- +# 主执行逻辑 +# --------------------- +def run_for_product(product_name, baseurl_map, filename_prefix, updates_test_url, show_raw=False): + print(f"\n🔍 正在分析产品：{product_name}") + wb = openpyxl.Workbook() + if show_raw: + raw_sheet = wb.create_sheet("仓库原始数据") + analysis_sheet = wb.create_sheet("仓库所有SRPM分析") + focus_sheet_external = wb.create_sheet("外网仓库本周重点分析的SRPM") + focus_sheet_test = wb.create_sheet("测试仓库本周重点分析的SRPM") + wb.remove(wb.active) + + all_pkg_info_by_repo = {} + if show_raw: + is_first_raw = True + for repo_name, baseurl in baseurl_map.items(): + info = get_package_info_from_db1(baseurl) + all_pkg_info_by_repo[repo_name] = info + write_Excel(info, repo_name, raw_sheet, is_first=is_first_raw) + is_first_raw = False + else: + for repo_name, baseurl in baseurl_map.items(): + all_pkg_info_by_repo[repo_name] = get_package_info_from_db1(baseurl) + + analysis_df = analyze_binary_distribution_all_repos(all_pkg_info_by_repo) + write_analysis_to_excel(analysis_df, "Merged", analysis_sheet, is_first=True) + + updates_url = baseurl_map.get("updates") + base_url = baseurl_map.get("base") + if updates_url and updates_test_url: + added_srpms, removed_srpms, added_bins, removed_bins, focus_srpms = compare_updates_and_test(updates_url, updates_test_url) + write_test_comparison_sheet(wb, added_srpms, removed_srpms, added_bins, removed_bins) + # 外网重点分析：base + updates + ext_info = { + 'base': get_package_info_from_db1(base_url), + 'updates': get_package_info_from_db1(updates_url) + } + ext_df = analyze_binary_distribution_all_repos(ext_info) + write_filtered_analysis_sheet(ext_df, focus_srpms, focus_sheet_external) + # 测试重点分析：base + updates_test + test_info = { + 'base': get_package_info_from_db1(base_url), + 'updates_test': get_package_info_from_db1(updates_test_url) + } + test_df = analyze_binary_distribution_all_repos(test_info) + write_filtered_analysis_sheet(test_df, focus_srpms, focus_sheet_test) + + timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M") + output_file = f"{filename_prefix}-{timestamp}.xlsx" + wb.save(output_file) + print(f"✅ {product_name} 分析完成，输出文件：{output_file}") + +# --------------------- +# 主函数 +# --------------------- +def main(): + products = load_product_config() + for name, conf in products.items(): + run_for_product( + product_name=name, + baseurl_map=conf['baseurls'], + filename_prefix=conf['filename'], + updates_test_url=conf.get('updates_test'), + show_raw=False + ) + +if __name__ == '__main__': + main() diff --git a/script/04_package_tps_test/config.ini b/script/04_package_tps_test/config.ini new file mode 100644 index 0000000..121cfe5 --- /dev/null +++ b/script/04_package_tps_test/config.ini @@ -0,0 +1,170 @@ +[V10SP4-x86] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/x86_64/ + updates=https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/updates/x86_64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/x86_64/ +filename = V10SP3-2403-x86分析报告 + +[V10SP3-x86] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/x86_64/ + updates=https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/updates/x86_64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/V10SP3/os/adv/lic/updates/x86_64/ +filename = V10SP3-x86分析报告 + +[V10SP2-x86] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/base/x86_64/ + updates=https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/updates/x86_64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/V10SP2/os/adv/lic/updates/x86_64/ +filename = V10SP2-x86分析报告 + +[V10SP1-x86] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/base/x86_64/ + updates=https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/updates/x86_64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/updates/x86_64/ +filename = V10SP1-x86分析报告 + +[HPC-x86] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/HPC/os/adv/lic/base/x86_64/ + updates=https://update.cs2c.com.cn/NS/V10/HPC/os/adv/lic/updates/x86_64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/HPC/os/adv/lic/updates/x86_64/ +filename = HPC-x86分析报告 + +[HOST-x86] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/HOST/SP3/os/adv/lic/base/x86_64/ + updates=https://update.cs2c.com.cn/NS/V10/HOST/SP3/os/adv/lic/updates/x86_64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/HOST/SP3/os/adv/lic/updates/x86_64/ +filename = HOST-x86分析报告 + + +[yundizuoV10-x86] +baseurls = + base=https://update.cs2c.com.cn/NS/HOST/2309/os/adv/lic/base/x86_64/ + updates=https://update.cs2c.com.cn/NS/HOST/2309/os/adv/lic/updates/x86_64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/HOST/2309/os/adv/lic/updates/x86_64/ +filename = yundizuoV10-x86分析报告 + + +[V10-ZJ-x86] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/V10-ZJ/os/adv/lic/base/x86_64/ + updates=https://update.cs2c.com.cn/NS/V10/V10-ZJ/os/adv/lic/updates/x86_64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/V10-ZJ/os/adv/lic/updates/x86_64/ +filename = V10-ZJ-x86分析报告 + +[2309a] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/2309A/os/adv/lic/base/aarch64/ + updates=https://update.cs2c.com.cn/NS/V10/2309A/os/adv/lic/updates/aarch64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/2309A/os/adv/lic/updates/aarch64/ +filename = 2309A分析报告 + + +[2309b] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/2309B/os/adv/lic/base/aarch64/ + updates=https://update.cs2c.com.cn/NS/V10/2309B/os/adv/lic/updates/aarch64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/2309B/os/adv/lic/updates/aarch64/ +filename = 2309B分析报告 + + +[V10SP4-aarch64] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/aarch64/ + updates=https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/updates/aarch64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/aarch64/ +filename = V10SP3-2403-aarch64分析报告 + +[V10SP3-aarch64] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/aarch64/ + updates=https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/updates/aarch64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/V10SP3/os/adv/lic/updates/aarch64/ +filename = V10SP3-aarch64分析报告 + +[V10SP2-aarch64] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/base/aarch64/ + updates=https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/updates/aarch64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/V10SP2/os/adv/lic/updates/aarch64/ +filename = V10SP2-aarch64分析报告 + +[V10SP1-aarch64] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/base/aarch64/ + updates=https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/updates/aarch64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/updates/aarch64/ +filename = V10SP1-aarch64分析报告 + +[HPC-aarch64] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/HPC/os/adv/lic/base/aarch64/ + updates=https://update.cs2c.com.cn/NS/V10/HPC/os/adv/lic/updates/aarch64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/HPC/os/adv/lic/updates/aarch64/ +filename = HPC-aarch64分析报告 + +[HOST-aarch64] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/HOST/SP3/os/adv/lic/base/aarch64/ + updates=https://update.cs2c.com.cn/NS/V10/HOST/SP3/os/adv/lic/updates/aarch64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/HOST/SP3/os/adv/lic/updates/aarch64/ +filename = HOST-aarch64分析报告 + + +[yundizuoV10-aarch64] +baseurls = + base=https://update.cs2c.com.cn/NS/HOST/2309/os/adv/lic/base/aarch64/ + updates=https://update.cs2c.com.cn/NS/HOST/2309/os/adv/lic/updates/aarch64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/HOST/2309/os/adv/lic/updates/aarch64/ +filename = yundizuoV10-aarch64分析报告 + + +[V10-ZJ-aarch64] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/V10-ZJ/os/adv/lic/base/aarch64/ + updates=https://update.cs2c.com.cn/NS/V10/V10-ZJ/os/adv/lic/updates/aarch64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/V10-ZJ/os/adv/lic/updates/aarch64/ +filename = V10-ZJ-aarch64分析报告 + + +[V10SP4-loongarch64] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/loongarch64/ + updates=https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/updates/loongarch64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/loongarch64/ +filename = V10SP3-2403-loongarch64分析报告 + +[V10SP3-loongarch64] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/loongarch64/ + updates=https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/updates/loongarch64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/V10SP3/os/adv/lic/updates/loongarch64/ +filename = V10SP3-loongarch64分析报告 + +[V10SP1-loongarch64] +baseurls = + base=https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/base/loongarch64/ + updates=https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/updates/loongarch64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/updates/loongarch64/ +filename = V10SP1-loongarch64分析报告 + +[V7-x86] +baseurls = + base= https://update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/base/x86_64/ + updates= https://update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/updates/x86_64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V7/V7Update9/os/adv/lic/updates/x86_64/ +filename = V7-x86分析报告 + +[V7-arm] +baseurls = + base= https://update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/base/aarch64/ + updates= https://update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/updates/aarch64/ +updates_test = https://update.cs2c.com.cn/private_test/repo/V7/V7Update9/os/adv/lic/updates/aarch64/ +filename = V7-arm分析报告 + + + diff --git a/script/04_package_tps_test/test_srpm_up_down.py b/script/04_package_tps_test/test_srpm_up_down.py new file mode 100644 index 0000000..c4b99e5 --- /dev/null +++ b/script/04_package_tps_test/test_srpm_up_down.py @@ -0,0 +1,137 @@ +# test_srpm_up_down.py +import os +from openpyxl import load_workbook +from rpm_vercmp import vercmp +from functools import cmp_to_key +import re + +def process_excel_to_dict(file_path, filter_choice, sheet_name): + wb = load_workbook(filename=file_path) + sheet = wb[sheet_name] + headers = [cell.value for cell in sheet[1]] + try: + name_idx = headers.index("SRPM Name") + version_idx = headers.index("SRPM Version") + binary_idx = headers.index("Binary NVR List") + changed_idx = headers.index("Binary Count Changed") + except ValueError as e: + print(f"Error: 缺少必要的列 - {e}") + return None + + result_dict = {} + for row in sheet.iter_rows(min_row=2, values_only=True): + if filter_choice == "Yes" and row[changed_idx] != "Yes": + continue + if filter_choice == "No" and row[changed_idx] != "No": + continue + name = row[name_idx] + version = row[version_idx] + binary_pkgs = row[binary_idx] + if name and version: + key = f"{name}-{version}" + result_dict[key] = binary_pkgs + return result_dict + +def process_single_file(file_path, filter_choice, sheet_name): + result = process_excel_to_dict(file_path, filter_choice, sheet_name) + if not result: + print(f"警告: 文件 {file_path} 处理失败，跳过") + return + + srpm_groups = {} + for key, value in result.items(): + parts = key.split('-') + name = '-'.join(parts[:-2]) + srpm_groups.setdefault(name, []).append((key, value)) + + for name in srpm_groups: + srpm_groups[name] = sorted(srpm_groups[name], key=cmp_to_key(lambda a, b: vercmp(a[0], b[0]))) + + base_name = re.sub(r'[^\x00-\x7F]+', '', os.path.basename(file_path)).split('.')[0] + output_dir = os.path.dirname(file_path) + script_name = os.path.join(output_dir, f"test_upgrade_downgrade_{base_name}.sh") + results_log = f"test_results_{base_name}.log" + process_log = f"test_process_{base_name}.log" + + try: + with open(script_name, 'w') as f: + f.write("#!/bin/bash\n\n") + f.write(f"# Auto-generated from {base_name}\n") + f.write(f"rm -f {results_log} {process_log}\n") + f.write(f"touch {process_log}\n") + f.write(f"exec > >(tee -a {process_log}) 2>&1\n") + f.write("echo 'Logging started at $(date)'\n") + f.write("echo '========================================'\n\n") + + for name, versions in srpm_groups.items(): + if len(versions) < 2: + continue + f.write(f"echo '\n=== Testing SRPM: {name} ==='\n") + + if sheet_name == "测试仓库本周重点分析的SRPM": + highest_idx = len(versions) - 1 + highest_ver = versions[highest_idx][0] + highest_pkgs = ' '.join(versions[highest_idx][1].split()) + for i in range(highest_idx): + from_ver = versions[i][0] + from_pkgs = ' '.join(versions[i][1].split()) + f.write(f"echo 'Installing base version: {from_ver}'\n") + f.write(f"if ! yum install -y {from_pkgs}; then\n") + f.write(f" echo 'BASE INSTALL FAILED: {from_ver}' >> {results_log}\n") + f.write(f"else\n") + f.write(f" echo 'BASE INSTALL SUCCESS: {from_ver}' >> {results_log}\n") + f.write(f" echo 'Testing upgrade: {from_ver} -> {highest_ver}'\n") + f.write(f" if yum upgrade -y {highest_pkgs}; then\n") + f.write(f" echo 'UPGRADE SUCCESS' >> {results_log}\n") + f.write(f" else\n") + f.write(f" echo 'UPGRADE FAILED' >> {results_log}\n") + f.write(f" fi\n") + f.write(f" echo 'Testing downgrade: {highest_ver} -> {from_ver}'\n") + f.write(f" if yum downgrade -y {from_pkgs}; then\n") + f.write(f" echo 'DOWNGRADE SUCCESS' >> {results_log}\n") + f.write(f" else\n") + f.write(f" echo 'DOWNGRADE FAILED' >> {results_log}\n") + f.write(f" fi\n") + f.write(f"fi\n") + f.write(f"echo '=== TEST END ===' >> {results_log}\n\n") + else: + for i in range(len(versions)): + for j in range(i+1, len(versions)): + from_ver = versions[i][0] + to_ver = versions[j][0] + from_pkgs = ' '.join(versions[i][1].split()) + to_pkgs = ' '.join(versions[j][1].split()) + f.write(f"echo 'Installing base version: {from_ver}'\n") + f.write(f"if ! yum install -y {from_pkgs}; then\n") + f.write(f" echo 'BASE INSTALL FAILED: {from_ver}' >> {results_log}\n") + f.write(f"else\n") + f.write(f" echo 'Testing upgrade: {from_ver} -> {to_ver}'\n") + f.write(f" if yum upgrade -y {to_pkgs}; then\n") + f.write(f" echo 'UPGRADE SUCCESS' >> {results_log}\n") + f.write(f" else\n") + f.write(f" echo 'UPGRADE FAILED' >> {results_log}\n") + f.write(f" fi\n") + f.write(f" echo 'Testing downgrade: {to_ver} -> {from_ver}'\n") + f.write(f" if yum downgrade -y {from_pkgs}; then\n") + f.write(f" echo 'DOWNGRADE SUCCESS' >> {results_log}\n") + f.write(f" else\n") + f.write(f" echo 'DOWNGRADE FAILED' >> {results_log}\n") + f.write(f" fi\n") + f.write(f"fi\n") + f.write(f"echo '=== TEST END ===' >> {results_log}\n\n") + print(f"✅ 成功生成测试脚本: {script_name}") + except Exception as e: + print(f"❌ 生成脚本失败: {e}") + +def gen_updown_test(excel_files, filter_choice="All", sheet_name="测试仓库本周重点分析的SRPM"): + if not excel_files: + print("错误: 未提供任何Excel文件") + return False + for file_path in excel_files: + if not os.path.exists(file_path): + print(f"警告: 文件不存在 {file_path}") + continue + print(f"\n处理文件: {os.path.basename(file_path)}") + process_single_file(file_path, filter_choice, sheet_name) + return True + diff --git a/script/04_package_tps_test/test_测试项2-ldy.sh b/script/04_package_tps_test/test_测试项2-ldy.sh new file mode 100644 index 0000000..b6f562b --- /dev/null +++ b/script/04_package_tps_test/test_测试项2-ldy.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# ------------------------- +# 读取命令行传入的产品名和架构 +# ------------------------- +if [ $# -ne 2 ]; then + echo "用法: $0 " + echo "示例: $0 V10 x86_64" + exit 1 +fi + +PRODUCT_NAME="$1" +ARCH="$2" + +# 软件包列表文件 +PACKAGE_LIST="Pkglist" + +# 自动生成输出文件名 +OUTPUT_FILE="测试项2-${PRODUCT_NAME}_${ARCH}_install_results-.csv" +LOG_FILE="测试项2-${PRODUCT_NAME}_${ARCH}_install_test-.log" + +# 创建或清空输出文件 +echo "Package,Install Status,Remove Status,Error" > "$OUTPUT_FILE" +echo "===== 安装测试日志开始 =====" > "$LOG_FILE" + +# 逐行读取软件包名称并安装 +while IFS= read -r package +do + if [ -z "$package" ]; then + continue + fi + + echo "正在安装: $package" | tee -a "$LOG_FILE" + + # 真正安装软件包 + install_output=$(yum --setopt=timeout=300 --setopt=retries=10 install -y "$package" 2>&1) + install_status=$? + + if [ $install_status -eq 0 ]; then + install_result="成功" + else + install_result="失败" + fi + + echo "$install_output" >> "$LOG_FILE" + + # 如果安装成功，执行卸载 + echo "正在卸载: $package" | tee -a "$LOG_FILE" + remove_output=$(yum remove -y "$package" 2>&1) + remove_status=$? + + if [ $remove_status -eq 0 ]; then + remove_result="成功" + else + remove_result="失败" + fi + + echo "$remove_output" >> "$LOG_FILE" + + # 写入 CSV（错误信息用双引号包裹防止格式错乱） + if [ "$install_result" = "失败" ]; then + error_msg="$install_output" + elif [ "$remove_result" = "失败" ]; then + error_msg="$remove_output" + else + error_msg="" + fi + + echo "$package,$install_result,$remove_result,\"$error_msg\"" >> "$OUTPUT_FILE" + + echo "--------------------------------" >> "$LOG_FILE" + +done < "$PACKAGE_LIST" + +echo "===== 安装测试日志结束 =====" >> "$LOG_FILE" +echo "测试完成，结果保存到 $OUTPUT_FILE，过程日志保存到 $LOG_FILE" diff --git a/script/05_src_multi_version_test/README.md b/script/05_src_multi_version_test/README.md new file mode 100644 index 0000000..21c9f3f --- /dev/null +++ b/script/05_src_multi_version_test/README.md @@ -0,0 +1,36 @@ +# 源码包多版本升降级测试 + +## 前置条件 +- 需要基于丁瑶脚本生成的表格文件 +- 确保执行环境已安装所需依赖库 + +## 使用步骤 +1. 执行测试脚本： +```bash +python3 test_srpm_up_down_0512.py +``` + +2. 根据提示进行选择： + - **第一步** 输入文件路径： + - 1) 多个表格文件：输入文件夹绝对路径 + - 2) 单个表格文件：输入文件绝对路径 + - **第二步** 筛选条件(Binary Count Changed列)： + - 1) 仅筛选"No"：多版本但包个数一致 + - 2) 仅筛选"Yes"：多版本但包个数不一致 + - 3) 不筛选：处理全部数据 + - **第三步** 选择工作表： + - 1) 测试仓库本周重点分析的SRPM + - 生成测试顺序：A→D, D→A, B→D, D→B, C→D, D→C + - 2) 仓库所有SRPM分析 + - 生成所有版本排列组合的升降级测试 + +## 后续操作 +- 执行完成后会生成测试脚本 +- 将脚本传输到测试机执行： +```bash +bash 测试脚本 +``` + +> **注意** +> 当前生成的测试脚本存在缺陷： +> 如果base install版本安装失败，后续升降级测试需手动确认 \ No newline at end of file diff --git a/script/05_src_multi_version_test/test_srpm_up_down_0512.py b/script/05_src_multi_version_test/test_srpm_up_down_0512.py new file mode 100644 index 0000000..18fbd6f --- /dev/null +++ b/script/05_src_multi_version_test/test_srpm_up_down_0512.py @@ -0,0 +1,232 @@ +import os +import openpyxl +from openpyxl import load_workbook +from rpm_vercmp import vercmp +from functools import cmp_to_key + +def process_excel_to_dict(file_path, filter_choice, sheet_name): + # 加载工作簿并选择指定工作表 + wb = load_workbook(filename=file_path) + sheet = wb[sheet_name] + + # 获取表头行，确定列索引 + headers = [cell.value for cell in sheet[1]] + try: + name_idx = headers.index("SRPM Name") + version_idx = headers.index("SRPM Version") + binary_idx = headers.index("Binary NVR List") + changed_idx = headers.index("Binary Count Changed") + except ValueError as e: + print(f"Error: 缺少必要的列 - {e}") + return None + + result_dict = {} + + # 从第二行开始处理数据 + for row in sheet.iter_rows(min_row=2, values_only=True): + if filter_choice == "Yes" and row[changed_idx] != "Yes": + continue + if filter_choice == "No" and row[changed_idx] != "No": + continue + + name = row[name_idx] + version = row[version_idx] + binary_pkgs = row[binary_idx] + + if name and version: # 确保关键字段不为空 + key = f"{name}-{version}" + result_dict[key] = binary_pkgs + + return result_dict + +def process_single_file(file_path, filter_choice, sheet_name): + result = process_excel_to_dict(file_path, filter_choice, sheet_name) + # 按SRPM名称分组 + srpm_groups = {} + for key, value in result.items(): + parts = key.split('-') + name = '-'.join(parts[:-2]) + if name not in srpm_groups: + srpm_groups[name] = [] + srpm_groups[name].append((key, value)) + + + # 在每个分组内按版本号排序 + for name in srpm_groups: + srpm_groups[name] = sorted(srpm_groups[name], + key=cmp_to_key(lambda a,b: vercmp(a[0], b[0]))) + + if not result: + print(f"警告: 文件 {file_path} 处理失败，跳过") + return + + # 从Excel文件名生成基础名(不含路径、扩展名和中文) + import re + base_name = re.sub(r'[^\x00-\x7F]+', '', os.path.basename(file_path)).split('.')[0] + + # 生成测试脚本 + script_name = f"test_upgrade_downgrade_{base_name}.sh" + results_log = f"test_results_{base_name}.log" + process_log = f"test_process_{base_name}.log" + + try: + with open(script_name, 'w') as f: + f.write("#!/bin/bash\n\n") + f.write(f"# Auto-generated SRPM upgrade/downgrade test script from {base_name}\n") + f.write("# Tests all version combinations for each SRPM\n") + f.write(f"# Results will be saved to {results_log}\n") + f.write(f"# Process details will be saved to {process_log}\n\n") + f.write(f"rm -f {results_log} {process_log}\n") + f.write(f"touch {process_log} \n") + f.write(f"exec > >(tee -a {process_log}) 2>&1\n") + f.write("echo 'Logging started at $(date)'\n") + f.write("echo '========================================'\n\n") + + + for name, versions in srpm_groups.items(): + if len(versions) < 2: + continue + + f.write(f"echo '\n=== Testing SRPM: {name} ==='\n") + + if sheet_name == "测试仓库本周重点分析的SRPM": + # 本周重点分析的特殊测试顺序：每个中间版本与最高版本测试 + highest_idx = len(versions) - 1 + highest_ver = versions[highest_idx][0] + highest_pkgs = ' '.join(versions[highest_idx][1].split()) + + for i in range(highest_idx): + from_ver = versions[i][0] + from_pkgs = ' '.join(versions[i][1].split()) + + # 1. 安装基本版本 + f.write(f"echo 'Installing base version: {from_ver}'\n") + f.write(f"if ! yum install -y {from_pkgs}; then\n") + f.write(f" echo 'BASE INSTALL FAILED: {from_ver} - VERSION: {from_ver} upgrade/downgrade test requires manual handling' >> {results_log}\n") + f.write(f" echo '=== TEST END ===' >> {results_log}\n\n") + f.write(f"else\n") + f.write(f" echo 'BASE INSTALL SUCCESS: {from_ver}' >> {results_log}\n") + + # 2. 升级到最高版本 + f.write(f"echo 'Testing upgrade: {from_ver} -> {highest_ver}'\n") + f.write(f"if yum upgrade -y {highest_pkgs}; then\n") + f.write(f" echo 'UPGRADE SUCCESS: {name} from {from_ver} to {highest_ver}' >> {results_log}\n") + f.write(f"else\n") + f.write(f" echo 'UPGRADE FAILED: {name} from {from_ver} to {highest_ver}' >> {results_log}\n") + f.write(f"fi\n") + + # 3. 降级回基本版本 + f.write(f"echo 'Testing downgrade: {highest_ver} -> {from_ver}'\n") + f.write(f"if yum downgrade -y {from_pkgs}; then\n") + f.write(f" echo 'DOWNGRADE SUCCESS: {name} from {highest_ver} to {from_ver}' >> {results_log}\n") + f.write(f"else\n") + f.write(f" echo 'DOWNGRADE FAILED: {name} from {highest_ver} to {from_ver}' >> {results_log}\n") + f.write(f"fi\n") + + f.write(f"echo '=== TEST END ===' >> {results_log}\n\n") + + f.write(f"fi\n") + + + else: + # 原有测试逻辑：所有版本组合测试 + for i in range(len(versions)): + for j in range(i+1, len(versions)): + from_ver = versions[i][0] + to_ver = versions[j][0] + from_pkgs = ' '.join(versions[i][1].split()) + pkgs = ' '.join(versions[j][1].split()) + + # 1. 安装基本版本 + f.write(f"echo 'Installing base version: {from_ver}'\n") + f.write(f"if ! yum install -y {from_pkgs}; then\n") + f.write(f" echo 'BASE INSTALL FAILED: {from_ver} - VERSION: {from_ver} upgrade/downgrade test requires manual handling' >> {results_log}\n") + f.write(f" echo '=== TEST END ===' >> {results_log}\n\n") + f.write(f"else\n") + f.write(f" echo 'BASE INSTALL SUCCESS: {from_ver}' >> {results_log}\n") + # 2. 升级测试 + f.write(f"echo 'Testing upgrade: {from_ver} -> {to_ver}'\n") + f.write(f"if yum upgrade -y {pkgs}; then\n") + f.write(f" echo 'UPGRADE SUCCESS: {name} from {from_ver} to {to_ver}' >> {results_log}\n") + f.write(f"else\n") + f.write(f" echo 'UPGRADE FAILED: {name} from {from_ver} to {to_ver}' >> {results_log}\n") + f.write(f"fi\n") + + # 3. 降级测试 + f.write(f"echo 'Testing downgrade: {to_ver} -> {from_ver}'\n") + f.write(f"if yum downgrade -y {' '.join(versions[i][1].split())}; then\n") + f.write(f" echo 'DOWNGRADE SUCCESS: {name} from {to_ver} to {from_ver}' >> {results_log}\n") + f.write(f"else\n") + f.write(f" echo 'DOWNGRADE FAILED: {name} from {to_ver} to {from_ver}' >> {results_log}\n") + f.write(f"fi\n") + + f.write(f"echo '=== TEST END ===' >> {results_log}\n\n") + f.write(f"fi\n") + + + print(f"成功生成测试脚本: {script_name}") + except Exception as e: + print(f"生成脚本失败: {e}") + +if __name__ == "__main__": + print("请选择模式:") + print("1. 手动输入单个Excel文件路径") + print("2. 批量处理文件夹下所有Excel文件") + + while True: + choice = input("请输入模式编号(1/2): ").strip() + if choice in ('1', '2'): + break + print("错误: 请输入1或2") + + print("\n请选择Binary Count Changed筛选条件:") + print("1. 仅筛选Yes") + print("2. 仅筛选No") + print("3. 不筛选(全部处理)") + + while True: + filter_choice = input("请输入筛选条件编号(1/2/3): ").strip() + if filter_choice == '1': + filter_choice = "Yes" + break + elif filter_choice == '2': + filter_choice = "No" + break + elif filter_choice == '3': + filter_choice = "All" + break + print("错误: 请输入1、2或3") + + print("\n请选择工作表:") + print("1. 测试仓库本周重点分析的SRPM") + print("2. 仓库所有SRPM分析") + + while True: + sheet_choice = input("请输入工作表编号(1/2): ").strip() + if sheet_choice == '1': + sheet_name = "测试仓库本周重点分析的SRPM" + break + elif sheet_choice == '2': + sheet_name = "仓库所有SRPM分析" + break + print("错误: 请输入1或2") + + if choice == '1': + while True: + file_path = input("请输入Excel文件的绝对路径: ").strip() + if os.path.exists(file_path): + break + print(f"错误: 文件 '{file_path}' 不存在，请重新输入") + process_single_file(file_path, filter_choice, sheet_name) + else: + while True: + dir_path = input("请输入包含Excel文件的文件夹路径: ").strip() + if os.path.isdir(dir_path): + break + print(f"错误: 文件夹 '{dir_path}' 不存在，请重新输入") + + for filename in os.listdir(dir_path): + if filename.endswith('.xlsx') or filename.endswith('.xls'): + file_path = os.path.join(dir_path, filename) + print(f"\n处理文件: {filename}") + process_single_file(file_path, filter_choice, sheet_name) diff --git a/script/06_dnf_repoclosure/README.md b/script/06_dnf_repoclosure/README.md new file mode 100644 index 0000000..e72f304 --- /dev/null +++ b/script/06_dnf_repoclosure/README.md @@ -0,0 +1,67 @@ +# 测试项9 - 仓库中软件依赖检测脚本说明 + +## 脚本简介 + +本脚本用于测试指定产品的仓库中软件包的依赖异常，使用dnf repoclosure工具检测仓库软件包的依赖。 +脚本执行的终端环境架构需要和测试仓库的架构一致，否则会有很多误报。所以按照架构（aarch64,x86_64,loongarch64,mips64el）有对应的测试脚本。 +## 使用方法 +1. 在x86_64的服务器终端执行： +`python3 test_pkg_repoclosure_x86_64.py` + +2. 在aarch64的服务器终端执行： +`python3 test_pkg_repoclosure_aarch64.py` + +3. 在loongarch64的服务器终端执行： +`python3 test_pkg_repoclosure_loongarch64.py` + +4. 在mips64el的服务器终端执行： +`python3 test_pkg_repoclosure_mips64.py` + +--- + +## 输入文件 +测试脚本中的变量command_pairs 变量中包含当前测试产品版本对应的命令内容。该变量的每项由三部分组成： +1. 描述信息（如产品版本及架构） + +2. 检查已发布仓库中软件包依赖情况的 Shell 命令 + +3. 检查本次更新仓库中软件包依赖情况的 Shell 命令 + +以 V10SP1.1 x86_64 为例，其 command_pairs 项如下： +```python +command_pairs = [ + ( + "V10SP1.1 x86_64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/base/x86_64/ --repofrompath updates,https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/base/x86_64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'' + ) +] +``` +--- + +## 输出文件 +执行脚本执行时，会输出对应架构的执行结果，输出文件分别为： +- `package_comparison_aarch64.md` +- `package_comparison_x86_64.md` +- `package_comparison_loongarch64.md` +- `package_comparison_mips64el.md` + +每个文件中： +- 结果1：为已发布仓库中依赖报错软件包列表， +- 结果2：为测试仓库中依赖报错软件包列表 +- 结果3：结果2中列表比结果1中列表多的软件包的包名，即测试仓库新增的报错包列表。结果为“无”，说明本周更新的包没有新增依赖报错，如果有包名，则需要确认是否为误报及查看有报错的原因。 +--- + +## 脚本流程说明 + +1. 执行dnf repoclosure 检查已发布的updates仓库是否有依赖报错，将结果输出到结果1; +2. 执行dnf repoclosure 检查本周更新的updates仓库是否有依赖报错，将结果输出到结果2; +3. 对比结果1和2，输出结果3 = 结果2依赖报错列表 - 结果2依赖报错列表；结果3是测试仓库新增的报错包列表；如不是‘无’，表示更新的包有依赖报错，需要排查是否为误报； +4. 将步骤1-3的结果输出到md文档中。 + +--- + +## 注意事项 + +- 需要在对应架构的服务器上执行对应的脚本，不同架构上也可以执行，但会有很多误报。 +--- diff --git a/script/06_dnf_repoclosure/test_pkg_repoclosure_aarch64.py b/script/06_dnf_repoclosure/test_pkg_repoclosure_aarch64.py new file mode 100644 index 0000000..6c21395 --- /dev/null +++ b/script/06_dnf_repoclosure/test_pkg_repoclosure_aarch64.py @@ -0,0 +1,143 @@ +import subprocess +import re +from datetime import datetime + + +# 功能：检查某个产品的仓库中软件包的依赖报错 +# 输入：依赖检测的bash命令 +# 返回值： 依赖报错软件包列表 +def get_package_list(command): + try: + result = subprocess.run(command, shell=True, capture_output=True, text=True, check=True) + output = result.stdout + pattern = r'package: (\S+)' + packages = re.findall(pattern, output) + return set(packages) + except subprocess.CalledProcessError as e: + print(f"执行命令时出错: {e.stderr}") + return set() + + +# 功能：将软件包的nvr处理后返回name字符串 +# 输入：字符串name-version-release +# 返回值： 字符串name +def extract_name(nvr_string): + # 匹配形如 name-version-release 格式，只提取 name 部分 + match = re.match(r"^(.*)-[^-]+-[^-]+$", nvr_string) + return match.group(1) if match else nvr_string + + +# 定义命令对列表，每个元素为 (组名, 命令1, 命令2) +command_pairs = [ + + ( + "NS7.9 aarch64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/updates/aarch64/ --repofrompath updates,https://update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/updates/aarch64/ --repofrompath updates,https://update.cs2c.com.cn/private_test/repo/V7/V7Update9/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10SP1.1 aarch64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/base/aarch64/ --repofrompath updates,https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10SP2 aarch64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/base/aarch64/ --repofrompath updates,https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/V10SP2/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10SP3 aarch64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/aarch64/ --repofrompath updates,https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/V10SP3/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10SP3 2403 aarch64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/aarch64/ --repofrompath updates,https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'' + ), + ( + "HOSTOS aarch64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/HOST/SP3/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/HOST/SP3/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/HOST/SP3/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/NS/V10/HOST/SP3/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'' + ), + ( + "HOSTOS 2309 aarch64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/HOST/2309/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/NS/HOST/2309/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/HOST/2309/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/HOST/2309/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'' + ), + ( + "HOSTOS 2406 aarch64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/HOST/2406/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/NS/HOST/2406/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/HOST/2309/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/HOST/2406/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10SP3 2309A aarch64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/2309A/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/NS/V10/2309A/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/2309A/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/2309A/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10SP3 2309B aarch64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/2309B/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/NS/V10/2309B/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/2309B/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/2309B/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10 HPC aarch64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/HPC/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/NS/V10/HPC/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/HPC/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/HPC/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10 aarch64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/V10-ZJ/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/NS/V10/V10-ZJ/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/V10-ZJ/os/adv/lic/base/aarch64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/V10-ZJ/os/adv/lic/updates/aarch64/ --check updates --all -n|grep \'package:\'' + ) +] + + +file_name = "package_comparison_aarch64.md" +with open(file_name, 'a') as f: + f.write(f"# 软件包依赖对比结果\n") + f.write(f"## 生成时间: {datetime.now()}\n\n") + +for index, (group_name, command1, command2) in enumerate(command_pairs, 1): + package_list1 = get_package_list(command1) + package_list2 = get_package_list(command2) + + more_packages = package_list2 - package_list1 + less_packages = package_list1 - package_list2 + + names1 = [extract_name(nvr) for nvr in less_packages] + names2 = [extract_name(nvr) for nvr in more_packages] + more_names = set(names2) - set(names1) + + with open(file_name, 'a') as f: + f.write(f"## {group_name} 对比结果\n\n") + f.write("### 已发布仓库中执行的依赖检查命令\n") + f.write("```bash\n") + f.write(command1 + "\n") + f.write("```\n\n") + f.write("### 测试仓库中执行的依赖检查命令\n") + f.write("```bash\n") + f.write(command2 + "\n") + f.write("```\n\n") + f.write("## 结果1：已发布仓库中依赖报错软件包列表\n") + if less_packages: + for package in sorted(less_packages): + f.write(f"- {package}\n") + else: + f.write("无\n") + f.write("\n## 结果2：测试仓库中依赖报错软件包列表\n") + if more_packages: + for package in sorted(more_packages): + f.write(f"- {package}\n") + else: + f.write("无\n") + f.write("\n## 结果3：结果2中列表比结果1中列表多的软件包的包名，即测试仓库新增的报错包列表\n") + if more_names: + for name in sorted(more_names): + f.write(f"- {name}\n") + else: + f.write("无\n") + f.write("\n---\n\n") + + print(f"{group_name} 对比结果已输出到 {file_name} 文件中。") + diff --git a/script/06_dnf_repoclosure/test_pkg_repoclosure_loongarch64.py b/script/06_dnf_repoclosure/test_pkg_repoclosure_loongarch64.py new file mode 100644 index 0000000..a8e7d26 --- /dev/null +++ b/script/06_dnf_repoclosure/test_pkg_repoclosure_loongarch64.py @@ -0,0 +1,96 @@ +import subprocess +import re +from datetime import datetime + + +# 功能：检查某个产品的仓库中软件包的依赖报错 +# 输入：依赖检测的bash命令 +# 返回值： 依赖报错软件包列表 +def get_package_list(command): + try: + result = subprocess.run(command, shell=True, capture_output=True, text=True, check=True) + output = result.stdout + pattern = r'package: (\S+)' + packages = re.findall(pattern, output) + return set(packages) + except subprocess.CalledProcessError as e: + print(f"执行命令时出错: {e.stderr}") + return set() + + +# 功能：将软件包的nvr处理后返回name字符串 +# 输入：字符串name-version-release +# 返回值： 字符串name +def extract_name(nvr_string): + # 匹配形如 name-version-release 格式，只提取 name 部分 + match = re.match(r"^(.*)-[^-]+-[^-]+$", nvr_string) + return match.group(1) if match else nvr_string + + +# 定义命令对列表，每个元素为 (组名, 命令1, 命令2) +command_pairs = [ + ( + "V10SP1.1 loongarch64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/base/loongarch64/ --repofrompath updates,https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/updates/loongarch64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/base/loongarch64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/updates/loongarch64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10SP3 loongarch64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/loongarch64/ --repofrompath updates,https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/updates/loongarch64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/loongarch64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/V10SP3/os/adv/lic/updates/loongarch64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10SP3 2403 loongarch64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/loongarch64/ --repofrompath updates,https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/updates/loongarch64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/loongarch64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/loongarch64/ --check updates --all -n|grep \'package:\'' + ) +] + + +with open(file_name, 'a') as f: + f.write(f"# 软件包依赖对比结果\n") + f.write(f"## 生成时间: {datetime.now()}\n\n") + +for index, (group_name, command1, command2) in enumerate(command_pairs, 1): + package_list1 = get_package_list(command1) + package_list2 = get_package_list(command2) + + more_packages = package_list2 - package_list1 + less_packages = package_list1 - package_list2 + + names1 = [extract_name(nvr) for nvr in less_packages] + names2 = [extract_name(nvr) for nvr in more_packages] + more_names = set(names2) - set(names1) + + with open(file_name, 'a') as f: + f.write(f"## {group_name} 对比结果\n\n") + f.write("### 已发布仓库中执行的依赖检查命令\n") + f.write("```bash\n") + f.write(command1 + "\n") + f.write("```\n\n") + f.write("### 测试仓库中执行的依赖检查命令\n") + f.write("```bash\n") + f.write(command2 + "\n") + f.write("```\n\n") + f.write("## 结果1：已发布仓库中依赖报错软件包列表\n") + if less_packages: + for package in sorted(less_packages): + f.write(f"- {package}\n") + else: + f.write("无\n") + f.write("\n## 结果2：测试仓库中依赖报错软件包列表\n") + if more_packages: + for package in sorted(more_packages): + f.write(f"- {package}\n") + else: + f.write("无\n") + f.write("\n## 结果3：结果2中列表比结果1中列表多的软件包的包名，即测试仓库新增的报错包列表\n") + if more_names: + for name in sorted(more_names): + f.write(f"- {name}\n") + else: + f.write("无\n") + f.write("\n---\n\n") + + print(f"{group_name} 对比结果已输出到 {file_name} 文件中。") + diff --git a/script/06_dnf_repoclosure/test_pkg_repoclosure_mips64.py b/script/06_dnf_repoclosure/test_pkg_repoclosure_mips64.py new file mode 100644 index 0000000..7808181 --- /dev/null +++ b/script/06_dnf_repoclosure/test_pkg_repoclosure_mips64.py @@ -0,0 +1,89 @@ +import subprocess +import re +from datetime import datetime + + +# 功能：检查某个产品的仓库中软件包的依赖报错 +# 输入：依赖检测的bash命令 +# 返回值： 依赖报错软件包列表 +def get_package_list(command): + try: + result = subprocess.run(command, shell=True, capture_output=True, text=True, check=True) + output = result.stdout + pattern = r'package: (\S+)' + packages = re.findall(pattern, output) + return set(packages) + except subprocess.CalledProcessError as e: + print(f"执行命令时出错: {e.stderr}") + return set() + + +# 功能：将软件包的nvr处理后返回name字符串 +# 输入：字符串name-version-release +# 返回值： 字符串name +def extract_name(nvr_string): + # 匹配形如 name-version-release 格式，只提取 name 部分 + match = re.match(r"^(.*)-[^-]+-[^-]+$", nvr_string) + return match.group(1) if match else nvr_string + + +# 定义命令对列表，每个元素为 (组名, 命令1, 命令2) +command_pairs = [ + ( + "V10SP1.1 mips64el 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/base/mips64el/ --repofrompath updates,https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/updates/mips64el/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/base/mips64el/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/updates/mips64el/ --check updates --all -n|grep \'package:\'' + ) +] + + + +file_name = "package_comparison_mips64el.md" + +with open(file_name, 'a') as f: + f.write(f"# 软件包依赖对比结果\n") + f.write(f"## 生成时间: {datetime.now()}\n\n") + +for index, (group_name, command1, command2) in enumerate(command_pairs, 1): + package_list1 = get_package_list(command1) + package_list2 = get_package_list(command2) + + more_packages = package_list2 - package_list1 + less_packages = package_list1 - package_list2 + + names1 = [extract_name(nvr) for nvr in less_packages] + names2 = [extract_name(nvr) for nvr in more_packages] + more_names = set(names2) - set(names1) + + with open(file_name, 'a') as f: + f.write(f"## {group_name} 对比结果\n\n") + f.write("### 已发布仓库中执行的依赖检查命令\n") + f.write("```bash\n") + f.write(command1 + "\n") + f.write("```\n\n") + f.write("### 测试仓库中执行的依赖检查命令\n") + f.write("```bash\n") + f.write(command2 + "\n") + f.write("```\n\n") + f.write("## 结果1：已发布仓库报错列表比测试仓库列表多的包\n") + if less_packages: + for package in sorted(less_packages): + f.write(f"- {package}\n") + else: + f.write("无\n") + f.write("\n## 结果2：测试仓库报错列表比已发布仓库列表多的包\n") + if more_packages: + for package in sorted(more_packages): + f.write(f"- {package}\n") + else: + f.write("无\n") + f.write("\n## 结果3：结果2中列表比结果1中列表多的软件包的包名\n") + if more_names: + for name in sorted(more_names): + f.write(f"- {name}\n") + else: + f.write("无\n") + f.write("\n---\n\n") + + print(f"{group_name} 对比结果已输出到 {file_name} 文件中。") + diff --git a/script/06_dnf_repoclosure/test_pkg_repoclosure_x86_64.py b/script/06_dnf_repoclosure/test_pkg_repoclosure_x86_64.py new file mode 100644 index 0000000..22646d0 --- /dev/null +++ b/script/06_dnf_repoclosure/test_pkg_repoclosure_x86_64.py @@ -0,0 +1,132 @@ +import subprocess +import re +from datetime import datetime + + +# 功能：检查某个产品的仓库中软件包的依赖报错 +# 输入：依赖检测的bash命令 +# 返回值： 依赖报错软件包列表 +def get_package_list(command): + try: + result = subprocess.run(command, shell=True, capture_output=True, text=True, check=True) + output = result.stdout + pattern = r'package: (\S+)' + packages = re.findall(pattern, output) + return set(packages) + except subprocess.CalledProcessError as e: + print(f"执行命令时出错: {e.stderr}") + return set() + + +# 功能：将软件包的nvr处理后返回name字符串 +# 输入：字符串name-version-release +# 返回值： 字符串name +def extract_name(nvr_string): + # 匹配形如 name-version-release 格式，只提取 name 部分 + match = re.match(r"^(.*)-[^-]+-[^-]+$", nvr_string) + return match.group(1) if match else nvr_string + + +# 定义命令对列表，每个元素为 (组名, 命令1, 命令2) +command_pairs = [ + ( + "NS7.9 x86_64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/updates/x86_64 --repofrompath updates,https://update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/updates/x86_64 --repofrompath updates,https://update.cs2c.com.cn/private_test/repo/V7/V7Update9/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10SP1.1 x86_64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/base/x86_64/ --repofrompath updates,https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/base/x86_64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10SP2 x86_64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/base/x86_64/ --repofrompath updates,https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/base/x86_64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/V10SP2/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10SP3 x86_64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/x86_64/ --repofrompath updates,https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/x86_64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/V10SP3/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10SP3 2403 x86_64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/x86_64/ --repofrompath updates,https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath base,https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/x86_64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'' + ), + ( + "HOSTOS x86_64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/HOST/SP3/os/adv/lic/base/x86_64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/HOST/SP3/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/HOST/SP3/os/adv/lic/base/x86_64/ --repofrompath=updates,https://update.cs2c.com.cn/NS/V10/HOST/SP3/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'' + ), + ( + "HOSTOS 2309 x86_64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/HOST/2309/os/adv/lic/base/x86_64/ --repofrompath=updates,https://update.cs2c.com.cn/NS/HOST/2309/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/HOST/2309/os/adv/lic/base/x86_64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/HOST/2309/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'' + ), + ( + "HOSTOS 2406 x86_64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/HOST/2406/os/adv/lic/base/x86_64/ --repofrompath=updates,https://update.cs2c.com.cn/NS/HOST/2406/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/HOST/2309/os/adv/lic/base/x86_64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/HOST/2406/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10 HPC x86_64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/HPC/os/adv/lic/base/x86_64/ --repofrompath=updates,https://update.cs2c.com.cn/NS/V10/HPC/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/HPC/os/adv/lic/base/x86_64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/HPC/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'' + ), + ( + "V10 86_64 对比", + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/V10-ZJ/os/adv/lic/base/x86_64/ --repofrompath=updates,https://update.cs2c.com.cn/NS/V10/V10-ZJ/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'', + 'yum clean all;dnf repoclosure --disablerepo="*" --repofrompath=base,https://update.cs2c.com.cn/NS/V10/V10-ZJ/os/adv/lic/base/x86_64/ --repofrompath=updates,https://update.cs2c.com.cn/private_test/repo/V10/V10-ZJ/os/adv/lic/updates/x86_64/ --check updates --all -n|grep \'package:\'' + ) +] + + +file_name = "package_comparison_aarch64.md" +with open(file_name, 'a') as f: + f.write(f"# 软件包依赖对比结果\n") + f.write(f"## 生成时间: {datetime.now()}\n\n") + +for index, (group_name, command1, command2) in enumerate(command_pairs, 1): + package_list1 = get_package_list(command1) + package_list2 = get_package_list(command2) + + more_packages = package_list2 - package_list1 + less_packages = package_list1 - package_list2 + + names1 = [extract_name(nvr) for nvr in less_packages] + names2 = [extract_name(nvr) for nvr in more_packages] + more_names = set(names2) - set(names1) + + with open(file_name, 'a') as f: + f.write(f"## {group_name} 对比结果\n\n") + f.write("### 已发布仓库中执行的依赖检查命令\n") + f.write("```bash\n") + f.write(command1 + "\n") + f.write("```\n\n") + f.write("### 测试仓库中执行的依赖检查命令\n") + f.write("```bash\n") + f.write(command2 + "\n") + f.write("```\n\n") + f.write("## 结果1：已发布仓库中依赖报错软件包列表\n") + if less_packages: + for package in sorted(less_packages): + f.write(f"- {package}\n") + else: + f.write("无\n") + f.write("\n## 结果2：测试仓库中依赖报错软件包列表\n") + if more_packages: + for package in sorted(more_packages): + f.write(f"- {package}\n") + else: + f.write("无\n") + f.write("\n## 结果3：结果2中列表比结果1中列表多的软件包的包名，即测试仓库新增的报错包列表\n") + if more_names: + for name in sorted(more_names): + f.write(f"- {name}\n") + else: + f.write("无\n") + f.write("\n---\n\n") + + print(f"{group_name} 对比结果已输出到 {file_name} 文件中。") + diff --git a/script/07_compat_repo_check/README.md b/script/07_compat_repo_check/README.md new file mode 100644 index 0000000..0ff35a1 --- /dev/null +++ b/script/07_compat_repo_check/README.md @@ -0,0 +1,22 @@ +1.˵ + ֿ⼶ļԼ⡢ +2.Ŀ¼ṹ + - compat + - compat.py + - repolist + - requirements + - README.md + compat.py Խű + repolist Ʒֿб +3.װ + pip3 -r requirements +4.й + python3 compat.py +5.˵ + ԱĿ¼£ + /root/result/common/sp1/x86_64/repo-compare-report-20250425114902/repo.html + ʹparser_repo_jsonļ +6.repolistṹ˵ + baseurlbaseַֿ + updateurlupdateַֿ + testurlַֿ \ No newline at end of file diff --git a/script/07_compat_repo_check/tools/compat.py b/script/07_compat_repo_check/tools/compat.py new file mode 100644 index 0000000..775cf04 --- /dev/null +++ b/script/07_compat_repo_check/tools/compat.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +"""compat.py""" + +import concurrent.futures +import pandas as pd +import configparser +import subprocess +import threading +from datetime import datetime +from itertools import zip_longest +import time +import shutil +import json +import os +import re + + + +config = configparser.ConfigParser() +config.read('repolist') + +RESULT_DIR = "/root/result/common/" +RESULT_DIR1 = "/root/result/migrate/" + +def prepare_dirs(dirs): + if os.path.exists(dirs): + shutil.rmtree(dirs) + os.makedirs(dirs) + +def test_task(section, arch): + baseurl = config.get(section, 'baseurl') + updateurl = config.get(section, 'updateurl') + testurl = config.get(section, 'testurl') + origurl = ','.join([baseurl, updateurl]) + work_dir = RESULT_DIR1 + section + '/' + arch + prepare_dirs(work_dir) + cmd = "/usr/bin/kylin_pkgchecker -output %s repo -l 2 -o %s -t %s -a %s" % (work_dir, updateurl, origurl, arch) + popen = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + text=True + ) + stdout, stderr = popen.communicate() + +sections = ['sp1', 'sp2', 'sp3', 'sp3-2403'] + +def test_repo_update(index): + start_index = index + 1 + orig_arch = config.get(sections[index], 'arch') + orig_arch_list = re.split(',', orig_arch) + orig_baseurl = config.get(sections[index], 'baseurl') + orig_updateurl = config.get(sections[index], 'updateurl') + orig_testurl = ','.join([orig_baseurl, orig_updateurl]) + for j in range(start_index,len(sections)): + target_baseurl = config.get(sections[j], 'baseurl') + target_updateurl = config.get(sections[j], 'updateurl') + target_arch = config.get(sections[j], 'arch') + target_testurl = ','.join([target_baseurl, target_updateurl]) + target_arch_list = re.split(',',target_arch) + arch_list = list(set(orig_arch_list) & set(target_arch_list)) + # print(arch_list) + for arch in arch_list: + work_dir = RESULT_DIR + # prepare_dirs(work_dir) + # print(sections[index],sections[j],arch) + cmd = "/usr/bin/kylin_pkgchecker -output %s repo -l 3 -o %s -t %s -a %s" % (work_dir, target_testurl, orig_testurl, arch) + popen = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + text=True + ) + stdout, stderr = popen.communicate() + +def test_repo_migrations(): + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(test_repo_update, section) for section in range(len(sections))] + + results = [future.result() for future in concurrent.futures.as_completed(futures)] + + +def parser_config(section): + arch = config.get(section,'arch') + + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(test_task, section, a) for a in arch.split(',')] + + results = [future.result() for future in concurrent.futures.as_completed(futures)] + + +def get_current_date(): + timestamp = time.time() + localtime = time.localtime(timestamp) + now = time.strftime("%Y%m%d", localtime) + return now + +def get_max_len(): + pass + +def write_excel(data): + col = ["产品", "架构", "依赖失败", "兼容失败", "源仓库", "目标仓库"] + whitedata = list(zip_longest(data[0], data[1], data[2] , list(set(data[3])), data[4] , data[5], fillvalue = None)) + + now = get_current_date() + excel_file = "仓库兼容性检测%s.xlsx" % now + sheet_name = "失败列表" + df = pd.DataFrame(whitedata, columns = col) + df.to_excel( + excel_file, + sheet_name = sheet_name, + index = False, + startrow = 0, + engine = "openpyxl" + ) + +def append_sheet_excel(data): + book = load_workbook() + sheet = book['失败列表'] + last_row = sheet.max_row + append_df = pd.DataFrame(data) + +def parser_repo_json(): + """ + {"兼容":0,"NA":1,"不兼容":2,"待分析":3} + {"依赖完整":0,"依赖缺失":1,"NA":"NA"} + :return: + """ + + #缺少依赖包列表 + requirefaild = [] + #兼容性失败包列表 + compatfaild = [] + data_list = [] + # outfile = "/opt/repotest/repo.json" + filename = "repo.json" + + for dirpath, dirname, files in os.walk("/var/www/html/output/20250425/result/"): + if filename in files: + outfile = os.path.join(dirpath, filename) + + with open(outfile) as f: + data = json.load(f) + for k,v in data.items(): + if k == 'repo_data': + for r in v: + if r["otherRequiresRet"] == 1: + requirefaild.append(r["name"]) + if r["fileRet"] == 2 or r["cmdRet"] == 2 or r["serviceRet"] == 2 or r["configRet"] == 2 or r["abiRet"] == 2 or r["apiRet"] == 2: + compatfaild.append(r["name"]) + if k == 'summary_data': + arch = v["sys_arch"] + origin_repo_url = v["origin_repo"] + target_repo_url = v["target_repo"] + + maxlen = max(len(requirefaild), len(compatfaild)) + archlist = [arch] * maxlen + origin_repo_url_list = [origin_repo_url] * maxlen + target_repo_url_list = [target_repo_url] * maxlen + product = ["Kylin-v7"] * maxlen + data_slist.append(product) + data_list.append(archlist) + data_list.append(requirefaild) + data_list.append(compatfaild) + data_list.append(origin_repo_url_list) + data_list.append(target_repo_url_list) + print(data_list) + write_excel(data_list) + +def test_single_repo(): + sections = config.sections() + prepare_dirs(RESULT_DIR) + for section in sections: + parser_config(section) + +if __name__ == "__main__": + test_single_repo() diff --git a/script/07_compat_repo_check/tools/repolist b/script/07_compat_repo_check/tools/repolist new file mode 100644 index 0000000..3b8640a --- /dev/null +++ b/script/07_compat_repo_check/tools/repolist @@ -0,0 +1,77 @@ +[sp1] +baseurl=https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/base/ +updateurl=https://update.cs2c.com.cn/NS/V10/V10SP1.1/os/adv/lic/updates/ +testurl=https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/updates/ +arch=x86_64,aarch64,mips64el,loongarch64 + +[sp2] +baseurl=https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/base/ +updateurl=https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/updates/ +testurl=https://update.cs2c.com.cn/private_test/repo/V10/V10SP2/os/adv/lic/updates/ +arch=x86_64,aarch64 + +[sp3] +baseurl=https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/ +updateurl=https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/updates/ +testurl=https://update.cs2c.com.cn/private_test/repo/V10/V10SP3/os/adv/lic/updates/ +arch=x86_64,aarch64,loongarch64 + +[sp3-2403] +baseurl=https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/ +updateurl=https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/updates/ +testurl=https://update.cs2c.com.cn/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/ +arch=x86_64,aarch64,loongarch64 + +[hpc] +baseurl=https://update.cs2c.com.cn/NS/V10/HPC/os/adv/lic/base/ +updateurl=https://update.cs2c.com.cn/NS/V10/HPC/os/adv/lic/updates/ +testurl=https://update.cs2c.com.cn/private_test/repo/V10/HPC/os/adv/lic/updates/ +arch=x86_64,aarch64 + +[zj] +baseurl=https://update.cs2c.com.cn/NS/V10/V10-ZJ/os/adv/lic/base/ +updateurl=https://update.cs2c.com.cn/NS/V10/V10-ZJ/os/adv/lic/updates/ +testurl=https://update.cs2c.com.cn/private_test/repo/V10/V10-ZJ/os/adv/lic/updates/ +arch=x86_64,aarch64 + +[2309A] +baseurl=https://update.cs2c.com.cn/NS/V10/2309A/os/adv/lic/base/ +updateurl=https://update.cs2c.com.cn/NS/V10/2309A/os/adv/lic/updates/ +testurl=https://update.cs2c.com.cn/private_test/repo/V10/2309A/os/adv/lic/updates/ +arch=aarch64 + +[2309B] +baseurl=https://update.cs2c.com.cn/NS/V10/2309B/os/adv/lic/base/ +updateurl=https://update.cs2c.com.cn/NS/V10/2309B/os/adv/lic/updates/ +testurl=https://update.cs2c.com.cn/private_test/repo/V10/2309B/os/adv/lic/updates/ +arch=aarch64 + +[HOST-2309] +baseurl=https://update.cs2c.com.cn/NS/HOST/2309/os/adv/lic/base/ +updateurl=https://update.cs2c.com.cn/NS/HOST/2309/os/adv/lic/updates/ +testurl=https://update.cs2c.com.cn/private_test/repo/HOST/2309/os/adv/lic/updates/ +arch=x86_64,aarch64 + +[HOST-2406] +baseurl=https://update.cs2c.com.cn/NS/HOST/2406/os/adv/lic/base/ +updateurl=https://update.cs2c.com.cn/NS/HOST/2406/os/adv/lic/updates/ +testurl=https://update.cs2c.com.cn/private_test/repo/HOST/2406/os/adv/lic/updates/ +arch=x86_64,aarch64 + +[HOST] +baseurl=https://update.cs2c.com.cn/NS/V10/HOST/SP3/os/adv/lic/base/ +updateurl=https://update.cs2c.com.cn/NS/V10/HOST/SP3/os/adv/lic/updates/ +testurl=https://update.cs2c.com.cn/private_test/repo/V10/HOST/SP3/os/adv/lic/updates/ +arch=x86_64,aarch64 + +[V7] +baseurl=https://update.cs2c.com.cn/NS/V17/V7Update9/os/adv/lic/base/ +updateurl=https://update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/updates/ +testurl=https://update.cs2c.com.cn/private_test/repo/V7/V7Update9/os/adv/lic/updates/ +arch=x86_64,aarch64 + +[8U8] +baseurl=https://update.cs2c.com.cn/NS/V10/8U8/os/adv/lic/BaseOS/x86_64/os/,https://update.cs2c.com.cn/NS/V10/8U8/os/adv/lic/AppStream/x86_64/os/ +updateurl=https://update.cs2c.com.cn/NS/V10/8U8/os/adv/lic/BaseOS-updates/x86_64/os/,https://update.cs2c.com.cn/NS/V10/8U8/os/adv/lic/AppStream-updates/x86_64/os/ +testurl=https://update.cs2c.com.cn/private_test/repo/V7/V7Update9/os/adv/lic/updates/ +arch=x86_64 diff --git a/script/07_compat_repo_check/tools/requirements b/script/07_compat_repo_check/tools/requirements new file mode 100644 index 0000000..b5ebe88 --- /dev/null +++ b/script/07_compat_repo_check/tools/requirements @@ -0,0 +1,8 @@ +pandas +configparser +subprocess +threading +itertools +shutil +time +json diff --git a/script/08_V10SPX_products_upgrade_test/.gitkeep b/script/08_V10SPX_products_upgrade_test/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/script/08_V10SPX_products_upgrade_test/README.md b/script/08_V10SPX_products_upgrade_test/README.md new file mode 100644 index 0000000..ad804e6 --- /dev/null +++ b/script/08_V10SPX_products_upgrade_test/README.md @@ -0,0 +1,22 @@ +# 产品间升级测试 + +## 前置条件 +- 确保测试机网络在重启后能自动连接 + +## 使用步骤 +1. 将脚本传输到测试机 +2. 执行升级工具： +```bash +python3 upgrade_tool_pri.py +``` + +3. 选择升级路径： + - 1) V10SP1.1 → V10SP2 → V10SP3 → V10SP3-2403 + - 2) (V10SP1.1/V10SP2) → V10SP3 → V10SP3-2403 + - 3) V10SP1.1直接升级V10SP2 + - 4) (V10SP1.1/V10SP2)直接升级V10SP3 + - 5) (V10SP1.1/V10SP2/V10SP3)直接升级V10SP3-2403 + +## 注意事项 +- 脚本会生成守护进程`upgrade_tool.service`，机器重启后仍会执行；如果单步升级，需要手动systemctl stop upgrade_tool.service +- 结果日志存放在`/var/log/`目录upgrade_tool_*.log diff --git a/script/08_V10SPX_products_upgrade_test/upgrade_tool_pri_0512.py b/script/08_V10SPX_products_upgrade_test/upgrade_tool_pri_0512.py new file mode 100644 index 0000000..982cd3c --- /dev/null +++ b/script/08_V10SPX_products_upgrade_test/upgrade_tool_pri_0512.py @@ -0,0 +1,465 @@ +import subprocess +import time +import sys +import shutil +import os +from datetime import datetime + +# 全局系统架构变量 +SYSTEM_ARCH = None + +def get_log_file_path(arch): + """获取当前升级路径对应的日志文件路径 + + Returns: + str: 日志文件路径 + """ + choice, _, _ = load_state() + if choice and choice in UPGRADE_PATHS: + return f"/var/log/upgrade_tool_path{choice}_{arch}.log" + return "/var/log/upgrade_tool.log" + +def log_message(message): + """记录日志信息并打印到控制台 + + Args: + message (str): 要记录的日志消息 + + Returns: + None + """ + log_file = get_log_file_path(SYSTEM_ARCH) + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + log_entry = f"[{timestamp}] {message}\n" + with open(log_file, "a") as f: + f.write(log_entry) + print(log_entry.strip()) # 输出带时间戳的日志格式 + +# 升级路径定义 +UPGRADE_PATHS = { + "1": ["V10SP2", "V10SP3", "V10SP3-2403"], + "2": ["V10SP3", "V10SP3-2403"], + "3": ["V10SP2"], + "4": ["V10SP3"], + "5": ["V10SP3-2403"], +} + +NKVERS_CHECK = { + "V10SP1.1": "SP1", + "V10SP2": "SP2", + "V10SP3": "SP3", + "V10SP3-2403": "Halberd", +} + +# 每个版本对应的 repo 文件 +REPO_CONTENTS = { + "V10SP2": """ +[base] +name=Base Repo V10SP2 +baseurl=https://10.44.16.185/NS/V10/V10SP2/os/adv/lic/base/x86_64/ +enabled=1 +gpgcheck=0 +sslverify=False + +[updates] +name=Updates Repo V10SP2 +baseurl=https://10.44.16.185/private_test/repo/V10/V10SP2/os/adv/lic/updates/$basearch/ +enabled=1 +gpgcheck=0 +sslverify=False +""", + + "V10SP3": """ +[base] +name=Base Repo V10SP3 +baseurl=https://10.44.16.185/NS/V10/V10SP3/os/adv/lic/base/x86_64/ +enabled=1 +gpgcheck=0 +sslverify=False + +[updates] +name=Updates Repo V10SP3 +baseurl=https://10.44.16.185/private_test/repo/V10/V10SP3/os/adv/lic/updates/$basearch/ +enabled=1 +gpgcheck=0 +sslverify=False +""", + + "V10SP3-2403": """ +[base] +name=Base Repo V10SP3-2403 +baseurl=https://10.44.16.185/NS/V10/V10SP3-2403/os/adv/lic/base/x86_64/ +enabled=1 +gpgcheck=0 +sslverify=False + +[updates] +name=Updates Repo V10SP3-2403 +baseurl=https://10.44.16.185/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/$basearch/ +enabled=1 +gpgcheck=0 +sslverify=False +""" +} + +YUM_REPO_DIR = "/etc/yum.repos.d" + + +def run_command(cmd, check=True): + """执行系统命令并记录输出 + + Args: + cmd (str): 要执行的命令 + check (bool): 如果为True，命令失败会退出程序 + + Returns: + str: 命令的完整输出 + """ + log_message(f"\n>>> 执行命令：{cmd}") + + # 使用Popen实时输出 + process = subprocess.Popen(cmd, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1) # 行缓冲 + + output_lines = [] + while True: + # 实时读取输出 + line = process.stdout.readline() + if not line and process.poll() is not None: + break + if line: + line = line.strip() + print(line) # 实时打印 + output_lines.append(line) + + # 检查返回码 + returncode = process.wait() + if returncode != 0 and check: + error_msg = f"❌ 命令执行失败：{process.stderr.read().strip()}" + log_message(error_msg) + sys.exit(1) + + full_output = "\n".join(output_lines) + log_message(f"命令完整输出：\n{full_output}") + return full_output + + +def get_system_arch(): + """获取系统架构并返回对应的repo架构字符串 + + Returns: + str: 系统架构(x86_64/aarch64/loongarch64) + """ + arch = run_command("uname -m").strip() + # 架构映射 + arch_map = { + "x86_64": "x86_64", + "aarch64": "aarch64", + "loongarch64": "loongarch64", + } + # 查找匹配的架构 + for key, value in arch_map.items(): + if key in arch: + return value + log_message(f"⚠️ 未知架构: {arch}, 默认使用x86_64") + return "x86_64" # 默认回退 + +def write_repo_file(version): + """写入指定版本的YUM源配置文件 + + Args: + version (str): 目标版本(V10SP2/V10SP3/V10SP3-2403) + + Returns: + None + """ + log_message(f"📦 配置 {version} 的 YUM 源") + + # 获取系统架构并替换URL + arch = get_system_arch() + repo_content = REPO_CONTENTS[version].replace("x86_64", arch) + + # 备份现有repo文件 + for f in os.listdir(YUM_REPO_DIR): + if f.endswith(".repo") and not f.endswith(".bak"): + src = os.path.join(YUM_REPO_DIR, f) + dst = os.path.join(YUM_REPO_DIR, f"{f}.bak") + if os.path.exists(dst): + os.remove(dst) + os.rename(src, dst) + log_message(f"🔁 已备份: {f} -> {f}.bak") + + # 写入新repo文件 + new_repo_path = os.path.join(YUM_REPO_DIR, f"{version}.repo") + with open(new_repo_path, "w") as f: + f.write(repo_content) + log_message(f"✅ 已写入新 repo 文件：{new_repo_path}") + + +def print_progress(current, total): + progress = int((current / total) * 50) + print(f"\n[{'=' * progress}{' ' * (50 - progress)}] {current}/{total}") + +def upgrade_to_version(version, current_idx=None, total_steps=None): + """执行升级到指定版本的操作 + + Args: + version (str): 目标版本 + current_idx (int, optional): 当前步骤索引 + total_steps (int, optional): 总步骤数 + + Returns: + None + """ + if current_idx is not None and total_steps is not None: + print_progress(current_idx, total_steps) + log_message(f"\n🔧 开始升级到版本 {version} ...") + # 在备份yum源前先执行系统更新 + log_message("🔄 正在执行系统更新(dnf -y update)...") + run_command("dnf -y update") + log_message("✅ 系统更新完成") + write_repo_file(version) + run_command("dnf clean all") + run_command("dnf makecache") + run_command("dnf -y update") + + # 在重启前验证版本升级是否成功 + log_message("🔍 验证升级结果...") + check_version(version) + + # 在dnf update成功后更新状态(仅多步升级) + if current_idx is not None and total_steps > 1: + choice, versions, _ = load_state() + save_state(choice, versions, current_idx + 1) + + log_message("🔁 系统即将重启...") + run_command("reboot", check=False) + + +def wait_for_reboot(timeout=300): + """等待系统重启并恢复上线 + + Args: + timeout (int): 超时时间(秒)，默认300秒 + + Returns: + None + """ + log_message("⏳ 等待系统重启并恢复上线...") + time.sleep(10) + elapsed = 0 + while elapsed < timeout: + try: + run_command("ping -c 1 127.0.0.1", check=False) + # 系统上线后重启网络服务 + try: + run_command("ifdown ens192 && ifup ens192", check=False) + log_message("🔄 网络服务已重启") + except: + log_message("⚠️ 网络服务重启失败，继续执行") + log_message("✅ 系统已上线") + time.sleep(10) + return + except: + time.sleep(5) + elapsed += 5 + log_message("❌ 系统未能在预期时间内恢复，请检查虚拟机状态") + sys.exit(1) + + +def check_version(expected): + """验证当前系统版本是否符合预期 + + Args: + expected (str): 预期的版本号 + + Returns: + None + """ + log_message("🔍 正在验证版本...") + keyword = NKVERS_CHECK.get(expected, "") + if not keyword: + print(f"❌ 未知版本: {expected}") + sys.exit(1) + + # 使用grep直接检查关键字 + try: + run_command(f"nkvers | grep -q '{keyword}'") + print(f"✅ 升级成功，版本特征 '{keyword}' 验证通过") + # 仅在最后一步升级完成后清除状态 + if os.path.exists(STATE_FILE): + with open(STATE_FILE, "r") as f: + lines = f.read().splitlines() + if len(lines) >= 2 and int(lines[1]) == len(lines[2:]): + clear_state() + except: + print(f"❌ 升级失败，缺少版本特征关键字: '{keyword}'") + sys.exit(1) + + +def print_menu(): + """打印升级路径选择菜单 + + Returns: + None + """ + log_message("\n请选择升级路径：") + print("1) V10SP1.1 -> V10SP2 -> V10SP3 -> V10SP3-2403") + print("2) (V10SP1.1/V10SP2)--> V10SP3 --> V10SP3-2403") + print("3) V10SP1.1直接升级 V10SP2") + print("4) (V10SP1.1/V10SP2)直接升级 V10SP3") + print("5) (V10SP1.1/V10SP2/V10SP3)直接升级 V10SP3-2403") + print("q) 退出") + + +STATE_FILE = "/var/lib/upgrade_tool.state" + +def save_state(choice, versions, current_idx): + """保存升级状态到文件 + + Args: + choice (str): 用户选择的升级路径 + versions (list): 升级路径版本列表 + current_idx (int): 当前步骤索引 + + Returns: + None + """ + with open(STATE_FILE, "w") as f: + f.write(f"# 用户选择的升级路径\n{choice}\n") + f.write(f"# 当前执行的步骤索引(从0开始)\n{current_idx}\n") + f.write("# 升级路径版本列表\n") + f.write("\n".join(versions) + "\n") + +def load_state(): + """从文件加载升级状态 + + Returns: + tuple: (choice, versions, current_idx) 或 (None, None, None) + """ + if not os.path.exists(STATE_FILE): + return None, None, None + with open(STATE_FILE, "r") as f: + # 跳过注释行，只读取实际数据行 + lines = [line for line in f.read().splitlines() + if not line.startswith("#")] + if len(lines) < 2: + return None, None, None + choice = lines[0] + current_idx = int(lines[1]) + versions = lines[2:] + return choice, versions, current_idx + +def clear_state(): + """清除升级状态文件 + + Returns: + None + """ + if os.path.exists(STATE_FILE): + os.remove(STATE_FILE) + +def install_as_service(): + """将脚本安装为系统服务 + + Returns: + None + """ + service_content = f"""[Unit] +Description=System Upgrade Tool +After=network.target +Wants=network.target + +[Service] +Type=simple +ExecStart=/usr/bin/python3 {os.path.abspath(__file__)} +# 仅在非正常退出时重启(不包含正常退出码0) +Restart=on-failure +RestartSec=10s +User=root + +[Install] +WantedBy=multi-user.target +""" + + service_path = "/etc/systemd/system/upgrade_tool.service" + with open(service_path, "w") as f: + f.write(service_content) + + run_command("systemctl daemon-reload") + run_command("systemctl enable upgrade_tool.service") + log_message("✅ 已安装为系统服务") + +def main(): + """主函数，处理升级流程 + + Returns: + None + """ + global SYSTEM_ARCH + SYSTEM_ARCH = get_system_arch() + # 确保日志目录存在 + os.makedirs(os.path.dirname(get_log_file_path(SYSTEM_ARCH)), exist_ok=True) + log_message("=== 升级工具启动 ===") + + if not shutil.which("nkvers"): + log_message("⚠️ 未找到 nkvers 命令，请确保它在 PATH 中") + sys.exit(1) + + # 如果是首次运行，安装为服务 + if not os.path.exists("/etc/systemd/system/upgrade_tool.service"): + install_as_service() + + # 检查是否有未完成的升级任务 + choice, versions, current_idx = load_state() + if choice and versions and current_idx < len(versions): + log_message(f"🔍 检测到未完成的升级任务，正在恢复...") + + # 先验证上一步的升级结果(如果有) + if current_idx > 0: + prev_version = versions[current_idx - 1] + log_message(f"⏳ 验证前一步升级结果: {prev_version}") + check_version(prev_version) + + # 立即保存状态，防止重启后丢失进度 + save_state(choice, versions, current_idx) + + # 继续后续升级步骤 + for version in versions[current_idx:]: + upgrade_to_version(version, current_idx, len(versions)) + wait_for_reboot() + return + if choice and versions and current_idx == len(versions): + clear_state() + log_message("\n🎉 全部升级步骤完成！\n") + sys.exit(0) # 明确退出防止服务重启 + + + while True: + print_menu() + choice = input("请输入选项编号: ").strip() + if choice.lower() == "q": + print("👋 已退出。") + sys.exit(0) # 明确退出防止服务重启 + elif choice in UPGRADE_PATHS: + versions = UPGRADE_PATHS[choice] + # 仅多步升级创建状态文件 + if len(versions) > 1: + save_state(choice, versions, 0) + + for idx, version in enumerate(versions): + upgrade_to_version(version, idx, len(versions)) + wait_for_reboot() + + if len(versions) > 1: + clear_state() + print("\n🎉 全部升级步骤完成！\n") + + else: + print("⚠️ 无效输入，请重新选择。") + +if __name__ == "__main__": + main() diff --git a/script/09_v10spx_repo-rpm_cmp/README.md b/script/09_v10spx_repo-rpm_cmp/README.md new file mode 100644 index 0000000..a24a440 --- /dev/null +++ b/script/09_v10spx_repo-rpm_cmp/README.md @@ -0,0 +1,21 @@ + +脚本简介 +本脚本用于测试增量或者全量V10SPx 系列产品仓库中软件包版本对比。 +读取 “每周仓库更新内测清单-20250513-1.xlsx”里的bug以及cve sheet里的 中的源码包全名、关联产品、涉及架构三列。 + +使用方法 + +python3 check_update_prod_final.py + + +输出文件 + + +output-0513-x86.csv output-0513-arm.csv output-0513-loong.csv +csv里面内容有：包名 版本信息 关联产品 涉及架构 对比内容 对比结果 测试用例版本 _调试信息 +筛选出对比结果为error,upgrade的内容，提交到每周仓库更新内测清单的测试报错汇总表格中 + + +注意事项 +1.check_update_prod_final.py中引入的repo_cmp_20250103.py和repodata.py，按照实际路径写入 +2.check_update_prod_final.py中file_path和sheet_name读取的文件按照实际路径填写 diff --git a/script/09_v10spx_repo-rpm_cmp/check_update_prod_final.py b/script/09_v10spx_repo-rpm_cmp/check_update_prod_final.py new file mode 100644 index 0000000..13aa7cc --- /dev/null +++ b/script/09_v10spx_repo-rpm_cmp/check_update_prod_final.py @@ -0,0 +1,350 @@ +# -*- coding:utf-8 -*- +# Time:2025/1/3 14:13 +# Author:GuoChao +# FileName:check_update.py + +import openpyxl +import rpm_vercmp +from repo_cmp_20250103 import plus,bubbleSort +import csv +from collections import defaultdict +from repodata import write_to_csv + + +def update_test_results(file_path, sheet_name): + """ + 读取Excel文件中指定工作表的数据，并更新测试结果。 + 现在返回的是列表嵌套字典，字典的键是列标题，值是具体数据。 + + :param file_path: Excel文件路径 + :param sheet_name: 工作表名称 + """ + # 打开Excel文件 + wb = openpyxl.load_workbook(file_path, data_only=True, read_only=True) + + # 选择指定工作表 + if sheet_name not in wb.sheetnames: + raise ValueError(f"工作表 '{sheet_name}' 不存在。") + + sheet = wb[sheet_name] # 使用工作表名称选择 + + # 假设第一行是标题（列名） + headers = [sheet.cell(row=1, column=col).value for col in range(1, sheet.max_column + 1)] + + # 存储每行数据的字典列表 + test_case_list = [] + + # 假设读取B, C, O列 + #columns_to_read = [2, 3, 13] # B列、C列、O列 + columns_to_read = [1, 2, 3] + for row in range(2, sheet.max_row + 1): # 从第二行开始读取（跳过标题行） + test_case_dict = {headers[col-1]: sheet.cell(row=row, column=col).value for col in columns_to_read} + test_case_list.append(test_case_dict) + + return test_case_list + +def process_dicts_in_list(dict_list): + """ + 遍历列表中的字典，并对每个字典的元素进行处理。 + + :param dict_list: 一个包含字典的列表 + :return: 处理后的字典列表 + """ + processed_list = [] + + # 遍历字典列表 + for dictionary in dict_list: + processed_dict = {} + + try: + # 提取源码包的版本信息 + #package_name = dictionary['源码包全名'].rsplit('-', 2)[0] # 提取包名 + package_name = dictionary['源码包全名'] and dictionary['源码包全名'].rsplit('-', 2)[0] or '未知包名' + #package_version = dictionary['源码包全名'].rsplit('-', 2)[1] + "-" + dictionary['源码包全名'].rsplit('-', 2)[2].rsplit('.', 3)[0] # 提取版本 + package_version = (dictionary['源码包全名'] and dictionary['源码包全名'].rsplit('-', 2)[1] + "-" + + dictionary['源码包全名'].rsplit('-', 2)[2].rsplit('.', 3)[0]) or '未知版本' + + except (IndexError, KeyError) as e: + # 错误处理：如果数据格式不正确或缺少关键字段，跳过该条记录 + print(f"错误处理字典 {dictionary}: {e}") + continue + + # 获取关联产品和涉及架构 + package_product = dictionary.get('关联产品', 'N/A') # 使用 get 防止 KeyError + package_arch = dictionary.get('涉及架构', 'N/A') + + # 将处理后的信息存储到字典中 + processed_dict['包名'] = package_name + processed_dict['版本信息'] = package_version + processed_dict['关联产品'] = package_product + processed_dict['涉及架构'] = package_arch + + # 将处理后的字典添加到列表中 + processed_list.append(processed_dict) + + return processed_list + + +def extract_full_version(srpm_name): + """从SRPM文件名中提取完整版本信息（包含所有必要部分）""" + if not srpm_name: + return None + try: + # 移除.src.rpm后缀 + base = srpm_name.replace('.src.rpm', '') + + # 分割架构部分 + if '.' in base: + main_part, _ = base.rsplit('.', 1) # 丢弃架构信息 + else: + main_part = base + + # 分割版本和发行号 + if '-' in main_part: + # 从右边找到第一个连字符分割 + parts = main_part.rsplit('-', 1) + version_part = parts[0].split('-')[-1] # 获取真正的版本部分 + release_part = parts[1] + return f"{version_part}-{release_part}" + return None + except Exception as e: + print(f"版本解析错误 [{srpm_name}]: {str(e)}") + return None + + +def process_dicts_in_list(dict_list): + """处理原始数据，使用新的版本提取方法""" + processed_list = [] + for dictionary in dict_list: + processed_dict = {} + try: + full_name = dictionary.get('源码包全名', '') + + # 提取包名（保留原始逻辑） + package_name = full_name.rsplit('-', 2)[0] if full_name else '未知包名' + + # 使用新方法提取版本 + package_version = extract_full_version(full_name) or '未知版本' + + processed_dict.update({ + '包名': package_name, + '版本信息': package_version, + '关联产品': dictionary.get('关联产品', 'N/A'), + '涉及架构': dictionary.get('涉及架构', 'N/A') + }) + processed_list.append(processed_dict) + except Exception as e: + print(f"处理记录出错: {str(e)}") + continue + return processed_list + +def check_version(processed_list, productinfo): + """版本对比逻辑：仅对比测试用例关联产品对应的仓库""" + for processed_dict in processed_list: + # 获取当前测试用例的关联产品名称 + current_product = processed_dict.get('关联产品', '未知产品') + + # 如果产品不存在或未配置仓库，跳过 + if current_product not in productinfo: + processed_dict.update({ + '对比内容': "无关联产品信息", + '对比结果': "N/A" + }) + continue + + # 仅获取当前关联产品对应的仓库数据 + pkglist, filename = productinfo[current_product]['repolist'] + + # 收集所有版本及所属产品（用于展示） + all_versions = {} + # 按产品线分组的版本（用于比较） + product_line_versions = defaultdict(list) + + # 仅遍历当前产品的仓库 + for pkg in filename: + if pkg['srcname'] == processed_dict['包名']: + ver = extract_full_version(pkg['sourcerpm']) + if ver: + # 获取该包所属的产品线（如V10SP3-2403） + product_line = pkg.get('product', 'unknown') + + # 收集展示数据 + if ver not in all_versions: + all_versions[ver] = set() + all_versions[ver].add(product_line) + + # 收集比较数据 + product_line_versions[product_line].append(ver) + + # ... 其余逻辑保持不变 ... + # 计算各产品线的最高版本 + max_versions = {} + for product_line, versions in product_line_versions.items(): + sorted_versions = bubbleSort(versions) + if sorted_versions: + max_versions[product_line] = sorted_versions[-1] + + # 执行版本比较 + test_case_ver = processed_dict['版本信息'] + result = "无匹配版本" + error_flag = False + all_higher = True # 假设所有产品线版本都比测试用例高 + + if max_versions: + for product_line, max_ver in max_versions.items(): + try: + cmp = rpm_vercmp.vercmp(test_case_ver, max_ver) + if cmp > 0: + error_flag = True # 发现任一产品线版本低于测试用例 + break + if cmp >= 0: + all_higher = False # 存在不高于的情况 + except Exception as e: + print(f"版本比较错误: {str(e)}") + + if error_flag: + result = "error,upgrade" + elif all_higher: + result = "downgrade" + else: + result = "equal" + + # 生成对比内容（保持原始顺序） + sorted_versions = bubbleSort(list(all_versions.keys())) + compared_content = [] + for ver in sorted_versions: + products = '|'.join(all_versions[ver]) + compared_content.append(f"{ver} ({products})") + + # 更新结果 + processed_dict.update({ + '对比内容': '\n'.join(compared_content) or "无匹配版本", + '对比结果': result, + '测试用例版本': test_case_ver, + '_调试信息': ' | '.join([f"{k}:{v}" for k, v in max_versions.items()]) # 可选调试字段 + }) + + return processed_list +def parse_product_from_url(url): + """ + 从仓库URL中解析产品名称（例如V10SP1） + 示例URL：https://update.cs2c.com.cn/NS/V10/V10SP1/os/adv/lic/base/x86_64/ + """ + parts = url.strip('/').split('/') + try: + v10_index = parts.index('V10') + if v10_index + 1 < len(parts): + return parts[v10_index + 1] + except ValueError: + pass + return '未知产品' +# 示例调用 +#file_path = r'C:/Users/lenovo.DESKTOP-JAA702O/Desktop/update内网源集成测试单-20241231.xlsx' +#file_path = r'C:/Users/Lenovo/Desktop/update内网源集成测试单-20250318.xlsx' +file_path = r'/root/mxh/每周仓库更新内测清单-20250513-1.xlsx' +#file_path = r'D:\Desktop\sp1.1loon.xlsx' +#file_path = r'D:\Desktop\0422.xlsx' +#sheet_name = 'bug【20250415功能测试】' # 工作表名称 +sheet_name = 'x86' +#sheet_name = 'cve【20250408】' +#sheet_name = 'bug【20250325测试结果统计】' # 工作表名称 +# 定义产品名称与对应 URL 列表的映射 +product_urls = { + "银河麒麟高级服务器操作系统 V10 SP3": [ + "https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/x86_64/", + "https://10.44.16.185/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/x86_64/" + # "https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/aarch64/", + # "https://10.44.16.185/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/aarch64/" + # "https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/loongarch64/", + # "https://10.44.16.185/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/loongarch64" + ], + "银河麒麟高级服务器操作系统 V10 SP2": [ + "https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/x86_64/", + "https://10.44.16.185/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/x86_64/", + "https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/x86_64/", + "https://10.44.16.185/private_test/repo/V10/V10SP3/os/adv/lic/updates/x86_64/" + # "https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/aarch64/", + # "https://10.44.16.185/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/aarch64/", + # "https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/aarch64/", + # "https://10.44.16.185/private_test/repo/V10/V10SP3/os/adv/lic/updates/aarch64/" + # "https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/loongarch64/", + # "https://10.44.16.185/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/loongarch64/", + # "https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/loongarch64/", + # "https://10.44.16.185/private_test/repo/V10/V10SP3/os/adv/lic/updates/loongarch64/" + ], + "银河麒麟高级服务器操作系统 V10 SP1": [ + "https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/x86_64/", + "https://10.44.16.185/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/x86_64/", + "https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/x86_64/", + "https://10.44.16.185/private_test/repo/V10/V10SP3/os/adv/lic/updates/x86_64/", + "https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/base/x86_64/", + "https://10.44.16.185/private_test/repo/V10/V10SP2/os/adv/lic/updates/x86_64/" + # "https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/aarch64/", + # "https://10.44.16.185/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/aarch64/", + # "https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/aarch64/", + # "https://10.44.16.185/private_test/repo/V10/V10SP3/os/adv/lic/updates/aarch64/", + # "https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/base/aarch64/", + # "https://10.44.16.185/private_test/repo/V10/V10SP2/os/adv/lic/updates/aarch64/", + # "https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/base/loongarch64/", + # "https://10.44.16.185/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/loongarch64/", + # "https://update.cs2c.com.cn/NS/V10/V10SP3/os/adv/lic/base/loongarch64/", + # "https://10.44.16.185/private_test/repo/V10/V10SP3/os/adv/lic/updates/loongarch64/" + ] +} + + +# 动态生成 products_info（关键修改部分） +products_info = {} +for product_name, urls in product_urls.items(): + pkglist_total = [] + filename_total = [] + for url in urls: + # 调用 plus 处理单个 URL，返回 (pkglist_part, filename_part) + pkglist_part, filename_part = plus(url) + + # 解析当前 URL 的产品名称（例如 V10SP1） + product = parse_product_from_url(url) + + # 为每个软件包字典添加 product 字段 + for pkg in filename_part: + pkg['product'] = product # 添加关键字段 + + # 合并数据 + filename_total.extend(filename_part) + pkglist_total.extend(pkglist_part) + + # 更新 products_info + products_info[product_name] = { + 'repolist': (pkglist_total, filename_total) + } + + +result = update_test_results(file_path, sheet_name) +processed_list=process_dicts_in_list(result) + +processed_list=check_version(processed_list,products_info) + + +def write_to_csv(processed_list, output_filename): + """ + 将处理后的数据写入 CSV 文件 + """ + # 动态收集所有字段名（保持顺序） + fieldnames = [] + seen = set() + for item in processed_list: + for key in item.keys(): + if key not in seen: + seen.add(key) + fieldnames.append(key) + + # 写入 CSV 文件 + with open(output_filename, 'w', newline='', encoding='utf-8') as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + for row in processed_list: + # 填充缺失字段为空值 + full_row = {key: row.get(key, "") for key in fieldnames} + writer.writerow(full_row) + print(f"数据已保存至: {output_filename}") +write_to_csv(processed_list,'output-0515-x86.csv') diff --git a/script/09_v10spx_repo-rpm_cmp/repo_cmp_20250103.py b/script/09_v10spx_repo-rpm_cmp/repo_cmp_20250103.py new file mode 100644 index 0000000..614228a --- /dev/null +++ b/script/09_v10spx_repo-rpm_cmp/repo_cmp_20250103.py @@ -0,0 +1,759 @@ +# -*- coding:utf-8 -*- +# Time:2025/1/3 9:16 +# Author:GuoChao +# FileName:repo_cmp_20250103.py +import requests +from lxml import etree +import rpm_vercmp +import xlsxwriter +import datetime +import re +import xlwt +import os +import subprocess +import logging +logging.captureWarnings(True) +from repodata import plus,getsrc + +#####获取软件包列表##### + +def openfile(*files): # 方法1：用于从文件中获取软件包列表 + pkglist = [] + for file in files: # 对多个文件进行处理 + with open(file, "r", encoding='utf-8') as output: # 打开文件 + for line in output.readlines(): # 依次读取每行 + if ".rpm" in line: # 去除非rpm包的影响 也可以修改为判断其他关键字 + pkglist.append(line.strip()) # 去除每行头尾的空白 + print('\n' + '###### 00 通过 Openfile 获取软件包列表完成，软件包数量:', len(pkglist), '#####') + return pkglist + + +def gethtml(*baseurls): #方法2：从html文件解析软件包列表 + head = { + "User-Agent": "Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 80.0.3987.122 Safari / 537.36"} + pkglist = [] + for baseurl in baseurls: # 对多个url进行处理 + response = requests.get(baseurl, headers=head,verify=False) + # 解决 requests.exceptions.SSLError 请求异常，SSL错误，证书认证失败问题进行解决 + html = response.text + html = etree.HTML(html) + a_links = html.xpath("//a") # 查找所有 a 标签 ,//a 代表 html 下所有 a,html/body/a 同理 + for item in a_links: + item1 = item.get('href') # 获取a标签某一属性 + if ".rpm" in item1: # 去除非rpm包的影响 也可以修改为判断其他关键字,备选.el8 + item1 = item1.replace("%2B", "+") # 对url不能识别”+“进行修正 + pkglist.append(item1) + print('\n' + '##### 00 通过 Gethtml 获取软件包列表完成，软件包数量:', len(pkglist), '#####') + return pkglist + + +#####对仓库软件包进行分析##### +def rjbfx(lab,listA): #对软件包构成进行分析 + + # 首先对总的列表进行分析 + + # 区分i686和非i686 + i686l=[x for x in listA if ".i686" in x] + fi686l=[x for x in listA if ".i686" not in x] + + # 区分mbs和非mbs软件包 + mbs1 = [x for x in listA if ".module" in x] + fmbs1 = [x for x in listA if ".module" not in x] + + print('\n' + '##### 01 通过 Rjbgfh 分析 ' + lab + ' 软件包完成，涉及软件包总数量:', len(listA), '#####') + print("\n" + "其中涉及i686相关包数量:", len(i686l), "\n" + "其中涉及非i686相关软件包数量:", len(fi686l)) + print("其中涉及模块包数量:", len(mbs1), "\n" + "其中涉及非模块包数量:", len(fmbs1)) + + # 其次对其中i686软件包进行处理 + mbs2 = [x for x in i686l if ".module" in x] + fmbs2 = [x for x in i686l if ".module" not in x] + print('\n' + '##### 01 通过 Rjbgfh 分析 '+lab+' i686 软件包完成，涉及软件包数量:', len(i686l), '#####') + print("其中涉及模块包数量:", len(mbs2),"\n" + "其中涉及非模块包数量:", len(fmbs2)) + + # 最后对其中非i686软件包进行处理 + mbs3 = [x for x in fi686l if ".module" in x] + fmbs3 = [x for x in fi686l if ".module" not in x] + print('\n' + '##### 01 通过 Rjbgfh 分析 '+lab+' 非i686 软件包完成，涉及软件包数量:', len(fi686l), '#####') + print("其中涉及模块包数量:", len(mbs3),"\n" + "其中涉及非模块包数量:", len(fmbs3)) + + return listA,mbs1,fmbs1,i686l,mbs2,fmbs2,fi686l,mbs3,fmbs3 + + +#####严格模式比较##### +def ygjgfl(pkglist): #对严格模式处理下的结果进行分类 + + name = [] + types = [] + arch = [] + + for pkg in pkglist: + name.append(pkg) + + if ".module" in pkg and ".ks8" in pkg: # 对软件包类型、以及是否产品化进行判断 + types.append("模块包(产品化)") + elif ".module" in pkg: + types.append("模块包") + elif ".ks8" in pkg: + types.append("非模块包(产品化)") + else: + types.append("非模块包") + + # 对软件包类型进行判断 + if ".i686" in pkg: + arch.append("i686") + elif ".x86_64" in pkg: + arch.append("x86_64") + elif ".noarch" in pkg: + arch.append("noarch") + elif ".aarch64" in pkg: + arch.append("aarch64") + elif ".src" in pkg: + arch.append("src") + else: + arch.append("N/A") + mks=types.count("模块包(产品化)")+types.count("模块包") + fmks=types.count("非模块包")+types.count("非模块包(产品化)") + return name,types,arch,mks,fmks + + +def comp1(labA,labB,listA1,listB1): #严格模式下比较 + + + listA=[] + for pkg in listA1: + # if ".oe" in pkg: + # pkg1 = pkg.replace(".oe1|oe2003sp4", "") + # elif ".ky10" in pkg: + # pkg1 = pkg.replace(".ky10|.p\d{2}.ky10", "") + # else: + # pkg1=pkg + pkg1=pkg.rsplit("-",1)[0]+"-"+pkg.rsplit("-",1)[-1].split(".",1)[0] + listA.append(pkg1) + listB = [] + for pkg in listB1: + pkg1 = pkg.rsplit("-", 1)[0] + "-" + pkg.rsplit("-", 1)[-1].split(".", 1)[0] + # if ".oe" in pkg: + # pkg1 = pkg.replace(".oe1|oe2003sp4", "") + # elif ".ky10" in pkg: + # pkg1 = pkg.replace(".ky10|.p\d{2}.ky10", "") + # else: + # pkg1 = pkg + listB.append(pkg1) + listA = list(set(listA)) + print("去重后A",len(listA)) + listB= list(set(listB)) + print("去重后B" ,len(listB)) + + + common_pkg = [x for x in listA if x in listB] # 整体ab都有 + incommon_pkg1 = [x for x in listA if x not in listB] # 整体a有b没有 + incommon_pkg2 = [x for x in listB if x not in listA] # 整体b有a没有 + + ABtong,leixing1,arch1,mk1,fmk1=ygjgfl(common_pkg) # 对相同包进行分类 + Ashao,leixing2,arch2,mk2,fmk2=ygjgfl(incommon_pkg2) # 对A的少包进行分类 B独有 + Aduo,leixing3,arch3,mk3,fmk3=ygjgfl(incommon_pkg1) # 对A多的包进行分类 A独有 + + print('\n'+"##### 02 严格模式 软件包对比开始 #####") + print(labA+" 与 "+labB+"相同的软件包数量：",len(ABtong),"\n"+'模块包：',leixing1.count("模块包(产品化)")+leixing1.count("模块包"),"\n"+'其中非模块包：',leixing1.count("非模块包")+leixing1.count("非模块包(产品化)")) + print(labA+" 比 "+labB+"少的软件包数量：",len(Ashao),"\n"+'模块包：',leixing2.count("模块包(产品化)")+leixing2.count("模块包"),"\n"+'其中非模块包：',leixing2.count("非模块包")+leixing2.count("非模块包(产品化)")) + print(labA+" 比 "+labB+"多的软件包数量：",len(Aduo),"\n"+'模块包：',leixing3.count("模块包(产品化)")+leixing3.count("模块包"),"\n"+'其中非模块包：',leixing3.count("非模块包")+leixing3.count("非模块包(产品化)")) + print("##### 02 严格模式 软件包对比完成 #####"+'\n') + return Aduo,leixing3,Ashao,leixing2,ABtong,leixing1,arch1,arch2,arch3,mk1,fmk1,mk2,fmk2,mk3,fmk3 + + + +#####宽松模式比较##### + +def bubbleSort(arr): # 对同一仓库的多版本进行冒泡排序# + n = len(arr) + for i in range(n): # 遍历所有数组元素 + for j in range(0, n - i - 1): # Last i elements are already in place + if rpm_vercmp.vercmp(arr[j], arr[j + 1]) == 1: + arr[j], arr[j + 1] = arr[j + 1], arr[j] + return arr + +def getnvra(data): # 获取软件包列表nvra + filename=[] + for i in data: + (n, v, r, a) = (i.rsplit('-', 2)[0], i.rsplit('-', 2)[1], i.rsplit('-', 2)[2].rsplit('.', 2)[0], i.rsplit('.', 2)[1]) # 切片方式获取软件包nvra + filename1 = {'name': n, 'version': v, 'release': r, 'arch': a} + # (n, v, r) = (i.rsplit('-', 2)[0], i.rsplit('-', 2)[1], i.rsplit('-', 2)[2].rsplit('.', 2)[0]) # 切片方式获取软件包nvr + # filename1 = {'name': n, 'version': v, 'release': r} + filename.append(filename1) + print('\n'+'##### 02 通过 getnvra 获取软件包nvr完成，软件包个数:',len(filename),'#####') + return filename + + +def relgfh(j): #对release进行统一规范化修改 + + if ".module" in j: # 对模块包进行统一修正 + j = j.replace(".module+", ".module_") # 对红帽模块、龙蜥包命名进行修改 + #j = j.rsplit('+', 2)[0] # 仅去除模块包release中 +10155+7f691889 字段 + j = j.split('_', 1)[0] # 去除模块包relases中 +/_el8.2.0+10155+7f691889 字段,去除模块el8.x平台影响 + result = re.search(".ks8.\d{2}", j) # 去除 ks/ns 产品化影响### + if result: + result = result.group() + j = j.replace(str(result), "") + + elif ".oe" in j: # 对oe进行统一修正 + #j = j.replace(".module+", ".module_") + #j = j.rsplit('+', 2)[0] # 仅去除模块包release中 +10155+7f691889 字段 + j = j.split('_', 1)[0] # 去除模块包relases中 +/_el8.2.0+10155+7f691889 字段,去除模块el8.x平台影响 + # result = re.search(".oe\d{8}", j) # 去除 ks/ns 产品化影响### + result = re.search(".oe1", j) # 去除 ks/ns 产品化影响### + if result: + result = result.group() + j = j.replace(str(result), "") + + else: # # 对非模块包进行统一修正 + j = j.replace("ky10", "") + result = re.search(".ks8.\d{2}|.ns7.\d{2}|.p\d{2}", j) # 去除 ks/ns/p 产品化影响### + if result: + result = result.group() + j = j.replace(str(result), "") + + return j + + + +def ksjgfl(namelist,data): + + pkglist = [] + types=[] + arch=[] + + for i in namelist: + rpmname = [] + for j in data: + if j["name"] == i: + rpmname.append(j["name"] + "-" + j["version"] + "-" + j["release"] + "." + j["arch"]+ "." + "rpm") + #rpmname.append(j["name"] + "-" + j["version"] + "-" + j["release"]) + rpmname = bubbleSort(rpmname) + #print(rpmname) + if ".module" in rpmname[-1] and ".ks8" in rpmname[-1]: + types.append("模块包(产品化)") + elif ".module" in rpmname[-1]: + types.append("模块包") + elif ".ks8" in rpmname[-1]: + types.append("非模块包(产品化)") + else: + types.append("非模块包") + + if ".i686" in rpmname[-1]: + arch.append("i686") + elif ".x86_64" in rpmname[-1]: + arch.append("x86_64") + elif ".noarch" in rpmname[-1]: + arch.append("noarch") + elif ".aarch64" in rpmname[-1]: + arch.append("aarch64") + elif ".src" in rpmname[-1]: + arch.append("src") + else: + arch.append("N/A") + pkglist.append("\n".join(rpmname)) + return namelist,pkglist,types,arch + +def comp2(labA,labB,listA,listB): + + data1 = getnvra(listA) + data2 = getnvra(listB) + + listA_keys = [] + listB_keys = [] + + for i in data1: + key = i["name"] + listA_keys.append(key) + for i in data2: + key = i["name"] + listB_keys.append(key) + + common_keys = [x for x in listA_keys if x in listB_keys] + icommon_keys1 = [x for x in listA_keys if x not in common_keys] + icommon_keys2 = [x for x in listB_keys if x not in common_keys] + + common_keys = list(set(common_keys)) + icommon_keys1 = list(set(icommon_keys1)) + icommon_keys2 = list(set(icommon_keys2)) + + tongming,pkglist1,leixing4,arch6 = ksjgfl(common_keys, data1) + Aqueshi1,Aqueshi2,leixing5,arch4 = ksjgfl(icommon_keys2,data2) + Bqueshi1,Bqueshi2,leixing6,arch5 = ksjgfl(icommon_keys1, data1) + + + shengjiang = [] + pkglist2 = [] + + # 对AB同名软件包进行处理 + for i in common_keys: + ccc = [] + for dict1 in data1: + if dict1["name"] == i: + ccc.append(dict1["name"] + "-" + dict1["version"] + "-" + dict1["release"]+ "." + dict1["arch"] + "." + "rpm") + ccc = bubbleSort(ccc) + list1_release = ccc[-1].rsplit('-', 2)[2].rsplit('.', 2)[0] + + release1 = relgfh(list1_release) + #print(release1) + list1_version = ccc[-1].rsplit('-', 2)[1] + '-' + release1 + list1_arch = ccc[-1].rsplit('.', 2)[1] + + ddd = [] + for dict2 in data2: + if dict2["name"] == i: + ddd.append(dict2["name"] + "-" + dict2["version"] + "-" + dict2["release"] + "." + dict2["arch"] + "." + "rpm") + ddd = bubbleSort(ddd) + list2_version = ddd[-1].rsplit('-', 2)[2].rsplit('.', 2)[0] + release2 = relgfh(list2_version) + #print(release2) + list2_version = ddd[-1].rsplit('-', 2)[1] + '-' + release2 + list2_arch = ddd[-1].rsplit('.', 2)[1] + pkglist2.append("\n".join(ddd)) + + if rpm_vercmp.vercmp(list1_version, list2_version) == 1 and list1_arch == list2_arch: + shengjiang.append("upgrade") + elif rpm_vercmp.vercmp(list1_version, list2_version) == 1 and list1_arch != list2_arch: + shengjiang.append("最高版本架构不同，无法比较") + elif rpm_vercmp.vercmp(list1_version, list2_version) == -1 and list1_arch == list2_arch: + shengjiang.append("downgrade") + elif rpm_vercmp.vercmp(list1_version, list2_version) == -1 and list1_arch != list2_arch: + shengjiang.append("最高版本架构不同，无法比较") + elif rpm_vercmp.vercmp(list1_version, list2_version) == 0 and list1_arch == list2_arch: + shengjiang.append("nochange") + elif rpm_vercmp.vercmp(list1_version, list2_version) == 0 and list1_arch != list2_arch: + shengjiang.append("最高版本架构不同，无法比较") + + print("\n" + labA + " 与 " + labB + "同名软件包升降级判断完成数量：", len(shengjiang)) + print(labA + " 与 " + labB + "同名软件包数：", len(tongming), '\n' + labA + " 较 " + labB + "缺失的软件包数：",len(Aqueshi1),"\n" + labB + " 较 " + labA + "缺失的软件包数：", len(Bqueshi1)) + + return tongming, pkglist1, pkglist2, shengjiang, Aqueshi1, Aqueshi2, Bqueshi1, Bqueshi2, leixing4, leixing5, leixing6, arch4, arch5, arch6 + + + + +def Genexcel1(A,B,list1,list2,list3,list4,list5,list6,list7,list8,list9,list11,list12,list13,list14,list15,list16,list21,list22,list23,list24,list25,list26,list27,list28,list31,list32,list33,list34,srclist0,srclist1,srclist2,srclist11,srclist21): + now = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') #当前时间 + fileName = A+"_"+B+"-"+u'%s-%s.xlsx' % ("仓库对比测试(郭超)",now) + #fileName = "仓库对比-identity_new1.xls" # + output_dir = "test_results/" + fileName + workbook = xlsxwriter.Workbook(output_dir) # 创建工作簿 + # 新增一个粗体格式 + bold = workbook.add_format({'bold': True}) + # 设置自动换行 + column_text_wrap = workbook.add_format() + column_text_wrap.set_text_wrap() + # 创建子表 + + worksheet4 = workbook.add_worksheet("结果汇总") + worksheet1 = workbook.add_worksheet("软件包差异情况对比-严格") + worksheet2 = workbook.add_worksheet("同名软件包版本变化情况对比-宽松") + worksheet3 = workbook.add_worksheet("软件包缺失情况-宽松") + worksheet5 = workbook.add_worksheet("软件包增多情况-宽松") + + #data1=[A+'较'+B+'多的软件包',A+'较'+B+'少的软件包',A+'较'+B+'少的软件包对应源码包',A+'较'+B+'相同的软件包','相同软件包类型'] + data1 = [A + '比' + B + '多的软件包','多的软件包对应类型' ,'多的软件包对应架构' ,A + '比' + B + '缺少的软件包', '缺少的软件包对应类型','缺少的软件包对应架构', A + '较' + B + '相同的软件包', + '相同的软件包对应类型','相同的软件包对应架构'] + data2=[A+"与"+B+"同名的软件包",A+"中的软件包版本",B+"中的软件包版本",A+"相对"+ B+"的升降级情况","对应类型","对应架构",A+"中对应源码包"] + data3=[A+"比"+B+"缺失的软件包名",A+"比"+B+"缺失的软件包","缺失的软件包对应类型","缺失的软件包对应架构","对应源码包","对应源码包(去重)"] + data4=["汇总项","总数量",'模块包数量','非模块包数量'] + data5=[A+"比"+B+"增多的软件包名",A+"较"+B+"增多的软件包","增多的软件包对应类型","增多的软件包对应架构","对应源码包","对应源码包(去重)"] + + worksheet1.write_row('A1',data1,bold) + worksheet2.write_row('A1', data2, bold) + worksheet3.write_row('A1', data3, bold) + worksheet4.write_row('A1', data4, bold) + worksheet5.write_row('A1', data5, bold) + col = ord('A') + + worksheet1.write_column(chr(col) + "2", list1) + worksheet1.write_column(chr(col + 1) + "2", list2) + worksheet1.write_column(chr(col + 2) + "2", list3) + worksheet1.write_column(chr(col + 3) + "2", list4) + worksheet1.write_column(chr(col + 4) + "2", list5) + worksheet1.write_column(chr(col + 5) + "2", list6) + worksheet1.write_column(chr(col + 6) + "2", list7) + worksheet1.write_column(chr(col + 7) + "2", list8) + worksheet1.write_column(chr(col + 8) + "2", list9) + + worksheet2.write_column(chr(col) + "2", list11) + worksheet2.write_column(chr(col + 1) + "2", list12) + worksheet2.write_column(chr(col + 2) + "2", list13) + worksheet2.write_column(chr(col + 3) + "2", list14) + worksheet2.write_column(chr(col + 4) + "2", list15) + worksheet2.write_column(chr(col + 5) + "2", list16) + worksheet2.write_column(chr(col + 6) + "2", srclist0) + + worksheet3.write_column(chr(col) + "2", list21) + worksheet3.write_column(chr(col + 1) + "2", list22) + worksheet3.write_column(chr(col + 2) + "2", list23) + worksheet3.write_column(chr(col + 3) + "2", list24) + worksheet3.write_column(chr(col + 4) + "2", srclist1) + worksheet3.write_column(chr(col + 5) + "2", srclist11) + + worksheet5.write_column(chr(col ) + "2", list25) + worksheet5.write_column(chr(col + 1) + "2", list26) + worksheet5.write_column(chr(col + 2) + "2", list27) + worksheet5.write_column(chr(col + 3) + "2", list28) + worksheet5.write_column(chr(col + 4) + "2", srclist2) + worksheet5.write_column(chr(col + 5) + "2", srclist21) + + worksheet4.write_column(chr(col) + "2", list31) + worksheet4.write_column(chr(col + 1) + "2", list32) + worksheet4.write_column(chr(col + 2) + "2", list33) + worksheet4.write_column(chr(col + 3) + "2", list34) + workbook.close() + + return output_dir + + +def main(labA,labB,data1,data2,filename1,filename2): + """"i686 x86_64 aarch64 ,module 非module 需要区分开，分别进行比较 + 首先对仓库软件包架构进行区分，不同架构的包又分为module 以及非moule，再进行比较 + 例如 i686 x86_64 aarch64 noarch in name ，只对 i686进行区分 + """ + #print(Openfile(labA,data1)) + #LISTA1A,Mbs1A,Fmbs1A,LISTA2A,Mbs2A,Fmbs2A,LISTA3A,Mbs3A,Fmbs3A=rjbgfh(Openfile(labA,data1)) + #LISTA1B,Mbs1B,Fmbs1B,LISTA2B,Mbs2B,Fmbs2B,LISTA3B,Mbs3B,Fmbs3B=rjbgfh(Openfile(labB,data2)) + + LISTA1A, Mbs1A, Fmbs1A, LISTA2A, Mbs2A, Fmbs2A, LISTA3A, Mbs3A, Fmbs3A = rjbfx(labA,data1) + LISTA1B, Mbs1B, Fmbs1B, LISTA2B, Mbs2B, Fmbs2B, LISTA3B, Mbs3B, Fmbs3B = rjbfx(labB,data2) + + Aduo, leixing3, Ashao, leixing2, ABtong, leixing1,arch1,arch2,arch3,mk1,fmk1,mk2,fmk2,mk3,fmk3= comp1(labA, labB, LISTA1A,LISTA1B) + + print('\n' + "##### 03 i686 模块 宽松模式(按包名统计) 软件包对比开始 #####") + Atongming, Apkglist1, Apkglist2, Ashengjiang,AAqueshi1,AAqueshi2,ABqueshi1,ABqueshi2,Aleixing4,Aleixing5,Aleixing6,Aarch4,Aarch5,Aarch6=comp2(labA+ ' 模块',labB+ ' 模块',Mbs2A,Mbs2B) + + print('\n' + "##### 04 i686 非模块 宽松模式(按包名统计) 软件包对比开始 #####") + Btongming, Bpkglist1, Bpkglist2, Bshengjiang, BAqueshi1, BAqueshi2, BBqueshi1, BBqueshi2, Bleixing4, Bleixing5, Bleixing6 ,Barch4,Barch5,Barch6= comp2(labA + ' 非模块', labB + ' 非模块', Fmbs2A, Fmbs2B) + + print('\n' + "##### 03 非i686 模块 宽松模式(按包名统计) 软件包对比开始 #####") + Ctongming, Cpkglist1, Cpkglist2, Cshengjiang,CAqueshi1,CAqueshi2,CBqueshi1,CBqueshi2,Cleixing4,Cleixing5,Cleixing6,Carch4,Carch5,Carch6=comp2(labA+ ' 模块',labB+ ' 模块',Mbs3A,Mbs3B) + + print('\n' + "##### 04 非i686 非模块 宽松模式(按包名统计) 软件包对比开始 #####") + Dtongming, Dpkglist1, Dpkglist2, Dshengjiang, DAqueshi1, DAqueshi2, DBqueshi1, DBqueshi2, Dleixing4, Dleixing5, Dleixing6,Darch4,Darch5,Darch6 = comp2(labA + ' 非模块', labB + ' 非模块', Fmbs3A, Fmbs3B) + + + + tongming=Atongming+Btongming+Ctongming+Dtongming + pkglist1=Apkglist1+Bpkglist1+Cpkglist1+Dpkglist1 + pkglist2=Apkglist2+Bpkglist2+Cpkglist2+Dpkglist2 + shengjiang=Ashengjiang+Bshengjiang+Cshengjiang+Dshengjiang + + Aqueshi1=AAqueshi1+BAqueshi1+CAqueshi1+DAqueshi1 #包名 + Aqueshi2=AAqueshi2+BAqueshi2+CAqueshi2+DAqueshi2 #包全名 + + + Bqueshi1=ABqueshi1+BBqueshi1+CBqueshi1+DBqueshi1 + Bqueshi2=ABqueshi2+BBqueshi2+CBqueshi2+DBqueshi2 + + leixing4=Aleixing4+Bleixing4+Cleixing4+Dleixing4 + leixing5=Aleixing5+Bleixing5+Cleixing5+Dleixing5 + leixing6=Aleixing6+Bleixing6+Cleixing6+Dleixing6 + + arch4=Aarch4+Barch4+Carch4+Darch4 + arch5=Aarch5+Barch5+Carch5+Darch5 + arch6=Aarch6+Barch6+Carch6+Darch6 + + srclist0=getsrc(pkglist1, filename1) + srclist1=getsrc(Aqueshi2, filename2) + srclist11 = list(set(srclist1)) + srclist2=getsrc(Bqueshi2, filename1) + srclist21 = list(set(srclist2)) + + huizong=[labA+'获取到软件包：','',labB+'获取到软件包：','','严格模式'+labA + " 与 " + labB + "同名软件包：",'严格模式'+labA+"缺失软件包：",'严格模式'+labA+"增加软件包：",'','宽松模式'+labA + " 与 " + labB + "同名软件包：",'宽松模式'+labA+"缺失软件包：",'宽松模式'+labA+"增加软件包："] + huizong1=[len(LISTA1A),'',len(LISTA1B),'',len(ABtong),len(Ashao),len(Aduo),'',len(tongming),len(Aqueshi1),len(Bqueshi1)] + huizong2=[len(Mbs1A),'',len(Mbs1B),'',mk1,mk2,mk3,'',len(Atongming)+len(Ctongming),len(AAqueshi1)+len(CAqueshi1),len(ABqueshi1)+len(CBqueshi1)] + huizong3=[len(Fmbs1A),'',len(Fmbs1B),'',fmk1,fmk2,fmk3,'',len(Btongming)+len(Dtongming),len(BAqueshi1)+len(DAqueshi1),len(BBqueshi1)+len(DBqueshi1)] + + resultname=Genexcel1(labA, labB, Aduo, leixing3,arch3, Ashao, leixing2,arch2,ABtong, leixing1,arch1,tongming, pkglist1, pkglist2, shengjiang, + leixing4,arch6, Aqueshi1, Aqueshi2, leixing5,arch4,Bqueshi1, Bqueshi2, leixing6,arch5,huizong,huizong1,huizong2,huizong3,srclist0,srclist1,srclist2,srclist11,srclist21) + + # return labA, labB, Aduo, leixing3,arch3, Ashao, leixing2,arch2,ABtong, leixing1,arch1,tongming, pkglist1, pkglist2, shengjiang,arch6,leixing4, Aqueshi1, Aqueshi2, leixing5,arch4,Bqueshi1, Bqueshi2, leixing6,arch5,huizong,huizong1,huizong2,huizong3 + + return resultname + + + +from bs4 import BeautifulSoup +#用于从网页获取软件包列表 +def Getpkglist(url): + head = {"User-Agent": "Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 80.0.3987.122 Safari / 537.36"} + # 查找所有标签 + response = requests.get(url) + soup = BeautifulSoup(response.content, 'html.parser') + PkgList = [] + packages = [] + directories = [] + + for a in soup.find_all('a', href=True): + href = a['href'] + if href.endswith('/'): + # 如果链接以斜杠结尾，表示是一个目录 + directories.append(href) + elif href.endswith('.rpm'): + # 如果链接以.rpm结尾，表示是一个软件包 + packages.append(href) + print(directories) + + directory_urls=[] + for directory in directories: + directory_url= '/'.join([url, directory]) + directory_urls.append(directory_url) + print(directory_urls) + + + for baseurl in directory_urls: + response = requests.get(baseurl, headers=head) + html = response.text + html = etree.HTML(html) + a_links = html.xpath("//a") # 查找所有 a 标签 ,//a 代表 html 下所有 a,html/body/a 同理 + for item in a_links: + item1 = item.get('href') # 获取a标签某一属性 + if ".rpm" in item1: # 去除非rpm包的影响，备选.el8 + item1 = item1.replace("%2B", "+") # 对url不能识别”+“进行修正 + PkgList.append(item1) + + print('\n'+'##### 00 通过 Gethtml 获取软件包列表完成，软件包数量:',len(PkgList),'#####') #检修口 + return PkgList + + +# repo1=("https://update.cs2c.com.cn/NS/V10/8U4/os/adv/lic/BaseOS/x86_64/","https://update.cs2c.com.cn/NS/V10/8U4/os/adv/lic/AppStream/x86_64/","https://update.cs2c.com.cn/NS/V10/8U4/os/adv/lic/PowerTools/x86_64/") +# repo2=("http://192.168.114.103/update.cs2c.com.cn/NS/V10/8U4/os/adv/lic/BaseOS/x86_64/","http://192.168.114.103/update.cs2c.com.cn/NS/V10/8U4/os/adv/lic/AppStream/x86_64/","http://192.168.114.103/update.cs2c.com.cn/NS/V10/8U4/os/adv/lic/PowerTools/x86_64/") + + +# data1,filename1=plus("https://10.44.16.185/private_test/history/repo/20231217/pub/V7/V7Update9/os/adv/lic/base/x86_64/ ")#,"https://update.cs2c.com.cn/NS/V10/8U4/os/adv/lic/AppStream/x86_64/") +# data2,filename2=plus("http://192.168.114.103/update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/base/x86_64/")#,"http://192.168.114.103/update.cs2c.com.cn/NS/V10/8U4/os/adv/lic/AppStream/x86_64/") + +#data2,filename2=plus("https://10.44.16.185/NS/V10/8U6/os/adv/lic/BaseOS-updates/x86_64/os/","https://10.44.16.185/NS/V10/8U6/os/adv/lic/AppStream-updates/x86_64/os/") + +#data2,filename2=plus("http://10.44.34.103/ns8.4/compose/BaseOS-updates/x86_64/os/","http://10.44.34.103/ns8.4/compose/AppStream-updates/x86_64/os/") +# data2,filename2=plus("https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/") +# +# #Baseos +# data1,filename1=plus("https://eps-server.openkylin.top/NS/V10/V10SP3/EPKL/main/x86_64/") +# data2,filename2=plus("https://10.44.16.185/private_test/history/repo/20231217/pub/V7/V7Update9/os/adv/lic/base/x86_64/","https://10.44.16.185/private_test/history/repo/20231217/pub/V7/V7Update9/os/adv/lic/base/i686/") +# https://eps-server.openkylin.top/NS/V10/V10SP3/EPKL/main/x86_64/ + + +# data1,filename1=plus("http://192.168.114.104/sp3-epkl-updates-3/source/") +# data1,filename1=plus("http://192.168.114.104/sp3-epkl-base/","http://192.168.114.104/sp3-epkl-updates-5/source/","http://192.168.114.104/sp3-epkl-220x/source/","http://192.168.114.104/sp3-epkl-230x/source/") +# data1,filename1=plus("http://192.168.114.104/sp1-epkl-base/","http://192.168.114.104/sp3-epkl-updates-3/source/") +# data2,filename2=plus("https://repo.openeuler.openatom.cn/openEuler-24.03-LTS/EPOL/main/source/","https://repo.openeuler.openatom.cn/openEuler-24.09/EPOL/main/source/") +# main("epkl-base+updates+22.0x+23.0x","epol-24.0x",data1,data2,filename1,filename2) + +# data2,filename2=plus("https://repo.openeuler.org/openEuler-20.03-LTS-SP1/EPOL/source/","https://repo.openeuler.org/openEuler-20.03-LTS-SP2/EPOL/main/source/","https://repo.openeuler.openatom.cn/openEuler-20.03-LTS-SP3/EPOL/main/source/","https://repo.openeuler.openatom.cn/openEuler-20.03-LTS-SP4/EPOL/main/source/") +# data2,filename2=plus("https://repo.openeuler.openatom.cn/openEuler-20.03-LTS-SP1/EPOL/update/source/","https://repo.openeuler.openatom.cn/openEuler-20.03-LTS-SP2/EPOL/update/main/source/","https://repo.openeuler.openatom.cn/openEuler-20.03-LTS-SP3/EPOL/update/main/source/","https://repo.openeuler.openatom.cn/openEuler-20.03-LTS-SP4/EPOL/update/main/source/") +# data2,filename2=plus("https://repo.openeuler.openatom.cn/openEuler-22.03-LTS/EPOL/main/source/","https://repo.openeuler.openatom.cn/openEuler-22.03-LTS-SP1/EPOL/main/source/","https://repo.openeuler.openatom.cn/openEuler-22.03-LTS-SP2/EPOL/main/source/","https://repo.openeuler.openatom.cn/openEuler-22.03-LTS-SP3/EPOL/main/source/","https://repo.openeuler.openatom.cn/openEuler-22.03-LTS-SP4/EPOL/main/source/") + +#epol 仓库地址 +# data2, filename2 = plus("https://repo.openeuler.org/openEuler-20.03-LTS-SP1/EPOL/source/") +# data2, filename2 = plus("https://repo.openeuler.org/openEuler-20.03-LTS-SP2/EPOL/main/source/") +# data2, filename2 = plus("https://repo.openeuler.openatom.cn/openEuler-20.03-LTS-SP3/EPOL/main/source/") +# data2, filename2 = plus("https://repo.openeuler.openatom.cn/openEuler-20.03-LTS-SP4/EPOL/main/source/") + +# data2, filename2 = plus("https://archives.openeuler.openatom.cn/openEuler-21.03/EPOL/source/") +# data2, filename2 = plus("https://archives.openeuler.openatom.cn/openEuler-21.09/EPOL/main/source/") + +# data2, filename2 = plus("https://repo.openeuler.openatom.cn/openEuler-22.03-LTS/EPOL/main/source/") +# data2, filename2 = plus("https://repo.openeuler.openatom.cn/openEuler-22.03-LTS-SP1/EPOL/main/source/") +# data2, filename2 = plus("https://repo.openeuler.openatom.cn/openEuler-22.03-LTS-SP2/EPOL/main/source/") +# data2, filename2 = plus("https://repo.openeuler.openatom.cn/openEuler-22.03-LTS-SP3/EPOL/main/source/") +# data2, filename2 = plus("https://repo.openeuler.openatom.cn/openEuler-22.03-LTS-SP4/EPOL/main/source/") +# data2, filename2 = plus("https://archives.openeuler.openatom.cn/openEuler-22.09/EPOL/main/source/") + +# data2, filename2 = plus("https://repo.openeuler.openatom.cn/openEuler-23.03/EPOL/main/source/") +# data2, filename2 = plus("https://repo.openeuler.openatom.cn/openEuler-23.09/EPOL/main/source/") + +# data2, filename2 = plus("https://repo.openeuler.openatom.cn/openEuler-24.03-LTS/EPOL/main/source/") +# data2, filename2 = plus("https://repo.openeuler.openatom.cn/openEuler-24.09/EPOL/main/source/") + +# data1,filename1=plus("http://192.168.114.104/sp3-epkl-base/","http://192.168.114.104/sp3-epkl-updates-4/source/") + +# data2, filename2 = plus("https://repo.openeuler.org/openEuler-20.03-LTS-SP1/EPOL/source/") +# data3, filename3 = plus("https://repo.openeuler.org/openEuler-20.03-LTS-SP3/EPOL/main/source/") +# data4, filename4 = plus("https://repo.openeuler.openatom.cn/openEuler-20.03-LTS-SP3/EPOL/main/source/") +# data5, filename5 = plus("https://repo.openeuler.openatom.cn/openEuler-20.03-LTS-SP4/EPOL/main/source/") +# #21 +# data6, filename6 = plus("https://archives.openeuler.openatom.cn/openEuler-21.03/EPOL/source/") +# data7, filename7 = plus("https://archives.openeuler.openatom.cn/openEuler-21.09/EPOL/main/source/") +# #22 +# data8, filename8 = plus("https://repo.openeuler.openatom.cn/openEuler-22.03-LTS/EPOL/main/source/") +# data9, filename9 = plus("https://repo.openeuler.openatom.cn/openEuler-22.03-LTS-SP1/EPOL/main/source/") +# data10, filename10 = plus("https://repo.openeuler.openatom.cn/openEuler-22.03-LTS-SP2/EPOL/main/source/") +# data11, filename11 = plus("https://repo.openeuler.openatom.cn/openEuler-22.03-LTS-SP3/EPOL/main/source/") +# data12, filename12 = plus("https://repo.openeuler.openatom.cn/openEuler-22.03-LTS-SP4/EPOL/main/source/") +# data13, filename13 = plus("https://archives.openeuler.openatom.cn/openEuler-22.09/EPOL/main/source/") +#23 +# data14, filename14 = plus("https://repo.openeuler.openatom.cn/openEuler-23.03/EPOL/main/source/") +# data15, filename15 = plus("https://repo.openeuler.openatom.cn/openEuler-23.09/EPOL/main/source/") + +#24 +# data16, filename16 = plus("https://repo.openeuler.openatom.cn/openEuler-24.03-LTS/EPOL/main/source/") +# data17, filename17 = plus("https://repo.openeuler.openatom.cn/openEuler-24.09/EPOL/main/source/") + + +#oepkg +# data18, filename18 = plus('https://repo.oepkgs.net/openEuler/rpm/openEuler-20.03-LTS/extras/source/',"https://repo.oepkgs.net/openEuler/rpm/openEuler-20.03-LTS-SP1/extras/source/","https://repo.oepkgs.net/openEuler/rpm/openEuler-20.03-LTS-SP2/extras/source/","https://repo.oepkgs.net/openEuler/rpm/openEuler-20.03-LTS-SP3/extras/source/","https://repo.oepkgs.net/openEuler/rpm/openEuler-20.03-LTS-SP4/extras/source/") +# data19, filename19 = plus("https://repo.oepkgs.net/openEuler/rpm/openEuler-20.03-LTS-SP1/extras/source/") +# data20, filename20 = plus("https://repo.oepkgs.net/openEuler/rpm/openEuler-20.03-LTS-SP2/extras/source/") +# data21, filename21 = plus("https://repo.oepkgs.net/openEuler/rpm/openEuler-20.03-LTS-SP3/extras/source/") +# data22, filename22 = plus("https://repo.oepkgs.net/openEuler/rpm/openEuler-20.03-LTS-SP4/extras/source/") + +# data23, filename23 = plus("https://repo.oepkgs.net/openEuler/rpm/openEuler-22.03-LTS/extras/source/","https://repo.oepkgs.net/openEuler/rpm/openEuler-22.03-LTS-SP1/extras/source/","https://repo.oepkgs.net/openEuler/rpm/openEuler-22.03-LTS-SP2/extras/source/","https://repo.oepkgs.net/openEuler/rpm/openEuler-22.03-LTS-SP3/extras/source/","https://repo.oepkgs.net/openEuler/rpm/openEuler-22.03-LTS-SP4/extras/source/","https://repo.oepkgs.net/openEuler/rpm/openEuler-24.03-LTS/extras/source/","https://repo.oepkgs.net/openEuler/rpm/openEuler-24.09/extras/source/","https://repo.oepkgs.net/openEuler/rpm/openEuler-24.09-LTS/extras/source/") +# data24, filename24 = plus("https://repo.oepkgs.net/openEuler/rpm/openEuler-22.03-LTS-SP1/extras/source/") +# data25, filename25 = plus("https://repo.oepkgs.net/openEuler/rpm/openEuler-22.03-LTS-SP2/extras/source/") +# data26, filename26 = plus("https://repo.oepkgs.net/openEuler/rpm/openEuler-22.03-LTS-SP3/extras/source/") +# data27, filename27 = plus("https://repo.oepkgs.net/openEuler/rpm/openEuler-22.03-LTS-SP4/extras/source/") +# data28, filename28 = plus("https://repo.oepkgs.net/openEuler/rpm/openEuler-24.03-LTS/extras/source/") +# data29, filename29 = plus("https://repo.oepkgs.net/openEuler/rpm/openEuler-24.09/extras/source/") +# data30, filename30 = plus("https://repo.oepkgs.net/openEuler/rpm/openEuler-24.09-LTS/extras/source/") + +# main("v10-sp3-epkl-base+updates","20.03-LTS-sp1-epol",data1,data2,filename1,filename2) +# main("v10-sp3-epkl-base+updates","20.03-LTS-sp2-epol",data1,data3,filename1,filename3) +# main("v10-sp3-epkl-base+updates","20.03-LTS-sp3-epol",data1,data4,filename1,filename4) +# main("v10-sp3-epkl-base+updates","20.03-LTS-sp4-epol",data1,data5,filename1,filename5) +# +# main("v10-sp3-epkl-base+updates","21.03-epol",data1,data6,filename1,filename6) +# main("v10-sp3-epkl-base+updates","21.09-epol",data1,data7,filename1,filename7) +# +# main("v10-sp3-epkl-base+updates","22.03-LTS-epol",data1,data8,filename1,filename8) +# main("v10-sp3-epkl-base+updates","22.03-LTS-sp1-epol",data1,data9,filename1,filename9) +# main("v10-sp3-epkl-base+updates","22.03-LTS-sp2-epol",data1,data10,filename1,filename10) +# main("v10-sp3-epkl-base+updates","22.03-LTS-sp3-epol",data1,data11,filename1,filename11) +# main("v10-sp3-epkl-base+updates","22.03-LTS-sp4-epol",data1,data12,filename1,filename12) +# main("v10-sp3-epkl-base+updates","22.09-epol",data1,data13,filename1,filename13) + +# main("v10-sp3-epkl-base+updates4","23.03-epol",data1,data14,filename1,filename14) +# main("v10-sp3-epkl-base+updates4","23.09-epol",data1,data15,filename1,filename15) +# main("v10-sp3-epkl-base+updates","24.03-epol",data1,data16,filename1,filename16) +# main("v10-sp3-epkl-base+updates","24.09-epol",data1,data17,filename1,filename17) + + +# main("v10-sp3-epkl-updates4","oepkg-20.03-LTS-SPX",data1,data23,filename1,filename18) +# main("v10-sp3-epkl-updates4","oepkg-22-24",data1,data23,filename23,filename23) +# main("oe-sp2-source+epol","oepkg-20.03-SP2",data3,data18,filename3,filename18) + + + + +#epol updates 仓库 +#20.03 +# data2,filename2=plus("https://repo.openeuler.openatom.cn/openEuler-20.03-LTS-SP1/EPOL/update/source/") +# data2,filename2=plus("https://repo.openeuler.openatom.cn/openEuler-20.03-LTS-SP2/EPOL/update/main/source/") +# data2,filename2=plus("https://repo.openeuler.openatom.cn/openEuler-20.03-LTS-SP3/EPOL/update/main/source/") +# data2,filename2=plus("https://repo.openeuler.openatom.cn/openEuler-20.03-LTS-SP4/EPOL/update/main/source/") +#21.0x +# data2,filename2=plus("https://archives.openeuler.openatom.cn/openEuler-21.03/EPOL/update/source/") +# data2,filename2=plus("https://repo.openeuler.openatom.cn/openEuler-20.03-LTS-SP4/EPOL/update/main/source/") + +# main("v10-epkl-base1+updates","22.03-lts-sp1-epol",data1,data2,filename1,filename2) +# main("v10-epkl-base1+updates","v10-epkl-base",data1,data2,filename1,filename2) + + +# data2,filename2=plus('https://dl.fedoraproject.org/pub/archive/fedora/linux/releases/33/Server/source/tree/','https://dl.fedoraproject.org/pub/archive/fedora/linux/releases/34/Server/source/tree/','https://dl.fedoraproject.org/pub/archive/fedora/linux/releases/35/Server/source/tree/','https://dl.fedoraproject.org/pub/archive/fedora/linux/releases/36/Server/source/tree/','https://dl.fedoraproject.org/pub/archive/fedora/linux/releases/37/Server/source/tree/','https://dl.fedoraproject.org/pub/archive/fedora/linux/releases/38/Server/source/tree/','https://dl.fedoraproject.org/pub/archive/fedora/linux/releases/39/Server/source/tree/') +# data2,filename2=plus('http://192.168.114.104/fedora/everything/f30/','http://192.168.114.104/fedora/everything/f31/','http://192.168.114.104/fedora/everything/f32/','http://192.168.114.104/fedora/everything/f33/','http://192.168.114.104/fedora/everything/f34/','http://192.168.114.104/fedora/everything/f35/','http://192.168.114.104/fedora/everything/f36/','http://192.168.114.104/fedora/everything/f37/','http://192.168.114.104/fedora/everything/f38/','http://192.168.114.104/fedora/everything/f39/') +# data3,filename3=plus("http://192.168.114.104/fedora/server/f30/") +# data2,filename2=plus("http://192.168.114.104/fedora/everything/f39/") +# main("oepkg-20.0x","fedora-server-30",data18,data3,filename18,filename3) +# main("oepkg-20.0x","fedora-everything-30-39",data18,data2,filename18,filename2) + +# data1,filename1=plus("https://dl.fedoraproject.org/pub/fedora/linux/releases/40/Everything/source/tree/") +# data2,filename2=plus("https://repo.oepkgs.net/openEuler/rpm/openEuler-24.03-LTS/fedora40/source/") +# data1,filename1=plus("http://192.168.114.104/fedora40/") +# data2,filename2=plus("http://192.168.114.104/oe-fedora40/") +# main("fedora40","fedora40",data1,data2,filename1,filename2) + +# data1,filename1=plus("https://repo.openeuler.org/openEuler-20.03-LTS-SP3/EPOL/main/source/","https://repo.openeuler.org/openEuler-20.03-LTS-SP3/EPOL/update/main/source/") +# data2,filename2=plus("https://repo.openeuler.org/openEuler-20.03-LTS-SP2/EPOL/main/source/","https://repo.openeuler.org/openEuler-20.03-LTS-SP2/EPOL/update/main/source/") +# data1,filename1=plus("https://repo.openeuler.org/openEuler-20.03-LTS-SP3/EPOL/main/source/","https://repo.openeuler.org/openEuler-20.03-LTS-SP3/EPOL/update/main/source/") +# data2,filename2=plus("https://repo.openeuler.org/openEuler-20.03-LTS-SP4/EPOL/main/source/","https://repo.openeuler.org/openEuler-20.03-LTS-SP4/EPOL/update/main/source/") + +#AppStream +#data1,filename1=plus("http://10.44.34.103/ns8.2-new/compose/b09/x86_64/BaseOS/","http://10.44.34.103/ns8.2-new/compose/b09/x86_64/AppStream/","http://10.44.34.103/ns8.2-new/compose/b09/x86_64/PowerTools/") +# data2,filename2=plus("http://192.168.114.15/iso/centos8.2/BaseOS/","http://192.168.114.15/iso/centos8.2/AppStream/") +#zong, +# 仅repodata +# data1,filename1=plus("http://10.44.34.103/ns8.2-new/compose/b09/x86_64/BaseOS/","http://10.44.34.103/ns8.2-new/compose/b09/x86_64/AppStream/","http://10.44.34.103/ns8.2-new/compose/b09/x86_64/PowerTools/") +#data1,filename1=plus("http://10.44.34.103/ns8.2-new/compose/b09/aarch64/BaseOS/","http://10.44.34.103/ns8.2-new/compose/b09/aarch64/AppStream/","http://10.44.34.103/ns8.2-new/compose/b09/aarch64/PowerTools/") + +# 全部仓库 +# data2,filename2=plus("http://10.44.34.103/ns8.2-new/compose/b17/BaseOS/x86_64/os/","http://10.44.34.103/ns8.2-new/compose/b17/AppStream/x86_64/os/","http://10.44.34.103/ns8.2-new/compose/b17/PowerTools/x86_64/os/") +#data1,filename1=plus("http://10.44.34.103/ns8.2-new/compose/b16/BaseOS/aarch64/os/","http://10.44.34.103/ns8.2-new/compose/b16/AppStream/aarch64/os/","http://10.44.34.103/ns8.2-new/compose/b16/PowerTools/aarch64/os/") +#data1,filename1=plus("http://10.44.34.103/ns8.8/b09/x86_64/BaseOS/","http://10.44.34.103/ns8.8/b09/x86_64/AppStream/","http://10.44.34.103/ns8.8/b09/x86_64/kernel418/") + +# data1,filename1=plus("https://update.cs2c.com.cn/NS/V10/8U8/os/adv/lic/BaseOS/x86_64/os/","https://update.cs2c.com.cn/NS/V10/8U8/os/adv/lic/AppStream/x86_64/os/","https://update.cs2c.com.cn/NS/V10/8U8/os/adv/lic/kernel418/x86_64/os/","https://update.cs2c.com.cn/NS/V10/8U8/os/adv/lic/kernel419/x86_64/os/") +# data2,filename2=plus("https://dl.rockylinux.org/vault/rocky/8.8/BaseOS/x86_64/os/","https://dl.rockylinux.org/vault/rocky/8.8/AppStream/x86_64/os/","https://dl.rockylinux.org/vault/rocky/8.8/PowerTools/x86_64/os/") +# #data1,filename1=plus("https://10.44.16.185/private_test/history/repo/20231226/pub/V10/8U6/os/adv/lic/AppStream-updates/x86_64/os/","https://10.44.16.185/private_test/history/repo/20231226/pub/V10/8U6/os/adv/lic/BaseOS-updates/x86_64/os/") +#data2,filename2=plus("http://192.168.114.15/iso/centos8.2/BaseOS/","http://192.168.114.15/iso/centos8.2/AppStream/","http://192.168.114.15/repo/centos82/powertool/") +# # #data2,filename2=plus("http://192.168.114.15/repo/centos82/baseos/","http://192.168.114.15/repo/centos82/appstrem/","http://192.168.114.15/repo/centos82/powertool/") +#data2,filename2=plus("https://10.44.16.185/NS/V10/8U2/os/adv/lic/BaseOS/x86_64/","https://10.44.16.185/NS/V10/8U2/os/adv/lic/AppStream/x86_64/","https://10.44.16.185/NS/V10/8U2/os/adv/lic/PowerTools/x86_64/")#"https://10.44.16.185/NS/V10/8U2/os/adv/lic/BaseOS-updates/x86_64/","https://10.44.16.185/NS/V10/8U2/os/adv/lic/AppStream-updates/x86_64/","https://10.44.16.185/NS/V10/8U2/os/adv/lic/PowerTools-updates/x86_64/","https://10.44.16.185/NS/V10/8U2/os/adv/lic/Plus-updates/x86_64/") +# 全部仓库 +#data2,filename2=plus("http://10.44.34.103/ns8.2-new/compose/b17/BaseOS/x86_64/os/","http://10.44.34.103/ns8.2-new/compose/b17/AppStream/x86_64/os/","http://10.44.34.103/ns8.2-new/compose/b17/PowerTools/x86_64/os/") +#data2,filename2=plus("http://10.44.34.103/ns8.2-new/compose/b17/BaseOS/aarch64/os/","http://10.44.34.103/ns8.2-new/compose/b17/AppStream/aarch64/os/","http://10.44.34.103/ns8.2-new/compose/b17/PowerTools/aarch64/os/") + +#data2,filename2=plus("http://10.44.51.209/centos-iso/centos8.2/BaseOS/","http://10.44.51.209/centos-iso/centos8.2/AppStream/","https://172.30.13.199/download/CENTOS8/centos8.2/PowerTools/aarch64/os/") +#data2,filename2=plus("https://10.44.16.185/private_test/repo/V10/8U4/os/adv/lic/AppStream-updates/x86_64/","https://10.44.16.185/private_test/repo/V10/8U4/os/adv/lic/BaseOS-updates/x86_64/") +# data1,filename1=plus("https://10.44.16.185/private_test/repo/V10/8U8/Build06/os/adv/lic/AppStream/x86_64/os/","https://10.44.16.185/private_test/repo/V10/8U8/Build06/os/adv/lic/BaseOS/x86_64/os/","https://10.44.16.185/private_test/repo/V10/8U8/Build06/os/adv/lic/kernel418/x86_64/os/","https://10.44.16.185/private_test/repo/V10/8U8/Build06/os/adv/lic/kernel419/x86_64/os/") +# #data2,filename2=plus("https://10.44.16.185/private_test/repo/V10/8U8/Build04/os/adv/lic/AppStream/aarch64/os/","https://10.44.16.185/private_test/repo/V10/8U8/Build04/os/adv/lic/BaseOS/aarch64/os/","https://10.44.16.185/private_test/repo/V10/8U8/Build04/os/adv/lic/kernel418/aarch64/os/") + +# data1,filename1=plus("https://10.44.16.185/private_test/repo/V10/8U8/Build06/os/adv/lic/BaseOS-updates/x86_64/os/","https://10.44.16.185/private_test/repo/V10/8U8/Build06/os/adv/lic/AppStream-updates/x86_64/os/") +# data2,filename2=plus("https://10.44.16.185/private_test/repo/V10/8U8/Build05/os/adv/lic/BaseOS-updates/x86_64/os/","https://10.44.16.185/private_test/repo/V10/8U8/Build05/os/adv/lic/AppStream-updates/x86_64/os/") + +# data2,filename2=plus("https://10.44.16.185/private_test/repo/V10/8U8/Build05/os/adv/lic/AppStream/x86_64/os/","https://10.44.16.185/private_test/repo/V10/8U8/Build05/os/adv/lic/BaseOS/x86_64/os/","https://10.44.16.185/private_test/repo/V10/8U8/Build05/os/adv/lic/kernel418/x86_64/os/","https://10.44.16.185/private_test/repo/V10/8U8/Build05/os/adv/lic/kernel419/x86_64/os/") +# # data2,filename2=plus("https://10.44.16.185/private_test/repo/V10/8U8/Build03/os/adv/lic/AppStream/aarch64/os/","https://10.44.16.185/private_test/repo/V10/8U8/Build03/os/adv/lic/BaseOS/aarch64/os/","https://10.44.16.185/private_test/repo/V10/8U8/Build03/os/adv/lic/kernel418/aarch64/os/") + +# pkglist=['nginx-filesystem-1.14.1-9.module_el8.6.0+647+66e644f1.ks8.01.noarch.rpm nginx-filesystem-1.16.1-2.module_el8.6.0+648+f70baf93.1.ks8.01.noarch.rpm nginx-filesystem-1.18.0-3.module_el8.6.0+649+52b76903.1.ks8.01.noarch.rpm nginx-filesystem-1.20.1-1.module_el8.6.0+707+e975214f.noarch.rpm nginx-filesystem-1.22.1-1.module_el8.8.0+948+03a664b8.noarch.rpm','cesi-fonts-1.0.0-1.el8.noarch.rpm'] +# print(getsrc(pkglist,filename1)) + +#data2=gethtml("http://10.44.51.209/guochao/rhel8.8/x86_64/AppStream/Packages/","http://10.44.51.209/guochao/rhel8.8/x86_64/BaseOS/Packages/") +#data2=gethtml("http://10.44.51.209/guochao/rhel8.8/aarch64/AppStream/Packages/","http://10.44.51.209/guochao/rhel8.8/aarch64/BaseOS/Packages/") +#data2=openfile('C:/Users/lenovo.DESKTOP-JAA702O/Desktop/123') + + +#data2=gethtml("https://mirrors.tuna.tsinghua.edu.cn/centos-vault/8.2.2004/AppStream/x86_64/os/Packages/","https://mirrors.tuna.tsinghua.edu.cn//centos-vault/8.2.2004/BaseOS/x86_64/os/Packages/","https://mirrors.tuna.tsinghua.edu.cn/centos-vault/8.2.2004/PowerTools/x86_64/os/Packages/") +#data2,filename2=plus("https://10.44.16.185/NS/V10/8U8/os/adv/lic/AppStream/x86_64/os/","https://10.44.16.185/NS/V10/8U8/os/adv/lic/BaseOS/x86_64/os/") +#data2,filename2=plus("https://10.44.16.185/NS/V10/8U8/os/adv/lic/AppStream/aarch64/os/","https://10.44.16.185/NS/V10/8U8/os/adv/lic/BaseOS/aarch64/os/") + + +# # data1,filename1=plus("https://update.cs2c.com.cn/NS/V10/8U8/os/adv/lic/AppStream/x86_64/os/")#e.cs2c.com.cn/NS/V10/8U4/os/adv/lic/AppStream/x86_64/") +# data1=gethtml("http://192.168.114.206/iso/rh8.10src/AppStream/Packages/")#, + +# main("rhel8.10-app-src","ns8.8-app-src",data1,data2,filename1,filename2) +# main("185-6.5","103-6.5",data1,data2,filename1,filename2) +# main("v10-sp3-epel","openeuler-20.03-sp4-epol",data1,data2,filename1,filename2) +# main("v10-sp3-epel","centos8-epel",data1,data2,filename1,filename2) +# main("ns7.9-0206","ns7.9-1217",data1,data2,filename1,filename2) +# main("ns8.4-updates-103","ns8.4-185",data1,data2,filename1,filename2) +# main("ns8.2-b01-arm-zong","centos8.2",data1,data2,filename1,filename2) + + + +# data2=gethtml("https://172.30.13.199/download/epel/epel/7/x86_64/Packages/") +# filename2="" + +# ###文件对比### +# data1=openfile("Kylin-Linux-Advanced-Server-v10-Release-Build06.12.06-lic-zj-package.txt") +# #data2=openfile('C:/Users/lenovo.DESKTOP-JAA702O/Desktop/ns8.2-baseos-ww.txt','C:/Users/lenovo.DESKTOP-JAA702O/Desktop/ns8.2-app-ww.txt') +# data2=openfile("nsV7Update6-adv-lic-build4-package.txt") +# filename1="" +# filename2="" +# main("v10-zj","v7.6",data1,data2,filename1,filename2) + +# data1,filename1=plus("https://update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/base/x86_64/","https://update.cs2c.com.cn/NS/V7/V7Update9/os/adv/lic/updates/x86_64/") +# data1,filename1=plus("http://192.168.114.102/v10.1/base/","http://192.168.114.102/v10.1/kernel/") +# data2=gethtml("https://vault.centos.org/7.9.2009/os/x86_64/Packages/","https://vault.centos.org/7.9.2009/updates/x86_64/Packages/") +# # main("ns8.10-061420-src","0619",data1,data2,filename1,filename2) +# filename2="" +# main("v10.1-cs2c","centos7.9-base+updates",data1,data2,filename1,filename2) + + + + +""" +过于繁琐 +函数改为自动判断、选择 +鲁棒性，容错性太低 +log输出 +结果输出格式需要易用性高 +""" diff --git a/script/09_v10spx_repo-rpm_cmp/repodata.py b/script/09_v10spx_repo-rpm_cmp/repodata.py new file mode 100644 index 0000000..3bdc8aa --- /dev/null +++ b/script/09_v10spx_repo-rpm_cmp/repodata.py @@ -0,0 +1,453 @@ +import bz2 +import lzma +import shutil +import sqlite3 +import requests +from lxml import etree +import logging +logging.captureWarnings(True) +import gzip +import os + +# def Getsqlite(baseurl): +# """ +# 从repodata中获取数据库压缩文件，下载并解压 +# url 格式 例如 basrutl=https://update.cs2c.com.cn/NS/V10/8U6/os/adv/lic/AppStream/x86_64/os/ +# """ +# head = { +# "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" +# } +# path = f"{baseurl}repodata/" +# try: +# response = requests.get(path, headers=head, verify=False) +# html = response.text +# html = etree.HTML(html) +# a_links = html.xpath("//a") +# +# for item in a_links: +# item1 = item.get('href') +# if item1.endswith("primary.sqlite.bz2"): +# sqlite_url = '/'.join([path, item1]) +# print(sqlite_url) +# response = requests.get(sqlite_url, headers=head, verify=False) +# un_path = "repotest.sqlite.bz2" +# with open(un_path, 'wb') as code: +# code.write(response.content) +# bz2file = bz2.BZ2File(un_path) +# data = bz2file.read() +# newfilepath = "repotest.sqlite" +# open(newfilepath, 'wb').write(data) +# return newfilepath +# +# elif item1.endswith("primary.sqlite.xz"): +# sqlite_url = '/'.join([path, item1]) +# print(sqlite_url) +# response = requests.get(sqlite_url, headers=head, verify=False) +# un_path = "repotest.sqlite.xz" +# +# with open(un_path, 'wb') as code: +# code.write(response.content) +# +# with lzma.open(un_path, 'rb') as input: +# with open("repotest.sqlite", 'wb') as output: +# shutil.copyfileobj(input, output) +# return "repotest.sqlite" +# +# elif item1.endswith("primary.sqlite.gz"): +# sqlite_url = '/'.join([path, item1]) +# print(sqlite_url) +# response = requests.get(sqlite_url, headers=head, verify=False) +# un_path = "repotest.sqlite.gz" +# +# with open(un_path, 'wb') as code: +# code.write(response.content) +# +# with gzip.open(un_path, 'rb') as input: +# with open("repotest.sqlite", 'wb') as output: +# shutil.copyfileobj(input, output) +# return "repotest.sqlite" +# +# print("获取数据库文件失败，请检查！") +# return None +# +# except Exception as e: +# print("发生异常：", e) +# return None + +def Getsqlite(baseurl): + """ + 从仓库 URL 下载并解压 repodata.sqlite 文件 + """ + import os # 确保在函数内导入（如果全局未导入） + head = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" + } + try: + # 确保 baseurl 以 '/' 结尾 + if not baseurl.endswith('/'): + baseurl += '/' + + # 正确拼接 repodata 路径 + repodata_url = f"{baseurl}repodata/" + response = requests.get(repodata_url, headers=head, verify=False) + response.raise_for_status() # 确保请求成功 + + html = etree.HTML(response.text) + a_links = html.xpath("//a") + + # 支持的压缩格式 + supported_formats = [ + ("primary.sqlite.bz2", bz2.BZ2File), + ("primary.sqlite.xz", lzma.open), + ("primary.sqlite.gz", gzip.open) + ] + + for item in a_links: + href = item.get('href', '') + for suffix, opener in supported_formats: + if href.endswith(suffix): + # 正确拼接下载 URL + sqlite_url = f"{repodata_url}{href}" + print(f"下载数据库文件: {sqlite_url}") + + # 下载压缩文件 + response = requests.get(sqlite_url, headers=head, verify=False) + response.raise_for_status() + + # 生成临时压缩文件路径 + temp_compressed = f"repotest.sqlite.{suffix.split('.')[-1]}" + with open(temp_compressed, 'wb') as f: + f.write(response.content) + + # 解压文件 + newfilepath = "repotest.sqlite" + with opener(temp_compressed, 'rb') as compressed_file: + with open(newfilepath, 'wb') as decompressed_file: + shutil.copyfileobj(compressed_file, decompressed_file) + + # 验证文件是否存在 + if not os.path.isfile(newfilepath): + raise FileNotFoundError(f"解压失败: {newfilepath} 不存在") + + return newfilepath # 返回有效路径 + + raise ValueError(f"未找到支持的数据库文件: {repodata_url}") + + except Exception as e: + print(f"获取数据库文件失败: {e}") + raise # 抛出异常供上层处理 +def Getsqlite1(baseurl): + """ + 从repodata中获取数据库压缩文件，下载并解压 + url 格式 例如 basrutl=https://update.cs2c.com.cn/NS/V10/8U6/os/adv/lic/AppStream/x86_64/os/ + """ + head = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" + } + path = f"{baseurl}repodata" + response = requests.get(path, headers=head,verify=False) + #print(response) + html = response.text + #print(html) + html = etree.HTML(html) + a_links = html.xpath("//a") # 查找所有 a 标签 ,//a 代表 html 下所有 a,html/body/a 同理 + + for item in a_links: + item1 = item.get('href') # 获取a标签某一属性 + if "primary.sqlite.bz2" or "primary.sqlite.xz" in item1: + sqlite_url = '/'.join([path, item1]) + #sqlite_url=f"{path}/{item1}"# 其他方法 + + response = requests.get(sqlite_url,headers=head,verify=False) + un_path = "repotest.sqlite" + if "primary.sqlite.bz2" in item1:#识别bz2压缩文件 + print(sqlite_url) + bz2path = 'repotest.sqlite.bz2' + #un_path = "repotest.sqlite" + with open(bz2path, 'wb') as code: + code.write(response.content) + bz2file = bz2.BZ2File(bz2path) + data = bz2file.read() + newfilepath = un_path + open(newfilepath, 'wb').write(data) + + elif "primary.sqlite.xz" in item1: #识别xz压缩文件 + print(sqlite_url) + xzfile = 'repotest.sqlite.xz' + #un_path = 'repotest.sqlite' + with open(xzfile, 'wb') as code: + code.write(response.content) + with lzma.open(xzfile, 'rb') as input: + with open(un_path, 'wb') as output: + shutil.copyfileobj(input, output) + else: + print("获取数据库文件失败，请检查！") + + return un_path + +# def Getrpminfo(database): +# """ +# 连接sqlite数据库获取软件包信息 +# """ +# con = sqlite3.connect(database) +# ##读取sqlite数据 +# cursor = con.cursor() +# ##创建游标cursor来执行executeＳＱＬ语句 +# cursor.execute("SELECT * from packages") +# # packs = cursor.fetchall() # 获取整个列表数据 +# content = cursor.execute("SELECT name, version, epoch, release, arch, rpm_sourcerpm, pkgId from packages") +# rpmname=[] +# filename1 = [] +# for row in cursor: +# rpm=row[0] + "-" + row[1] + "-" + row[3] + "." + row[4] + ".rpm" +# srcname=row[5].rsplit('-',2)[0] +# #rpmname.append(row[0] + "-" + row[1] + "-" + row[3] + "." + row[4] + ".rpm") +# #(n, v, e, r, a, s, i) = (row[0], row[1], row[2], row[3], row[4], row[5], row[6]) +# # filename = {'rpmname': rpm, 'name': row[0], 'version': row[1], 'epoch': row[2], 'release': row[3], 'arch': row[4], 'sourcerpm': row[5], 'pkgId': row[6]} +# filename = {'rpmname':rpm,'name':row[0], 'version': row[1], 'epoch':row[2],'release': row[3], 'arch': row[4],'sourcerpm':row[5],'pkgId': row[6],'srcname': srcname} +# filename1.append(filename) +# #print('\n' + '##### 通过 Getrpminfo 获取软件包 获取软件包nvr：', len(filename1)) # 检修口 +# #print(len(PkgList), filename2) +# return filename1 +# cursor.close() +# con.close() + +def Getrpminfo(database): + """ + 连接sqlite数据库获取软件包信息 + """ + if not os.path.isfile(database): + raise FileNotFoundError(f"数据库文件不存在: {database}") + + try: + con = sqlite3.connect(database) + cursor = con.cursor() + cursor.execute("SELECT name, version, epoch, release, arch, rpm_sourcerpm, pkgId from packages") + + filename1 = [] + for row in cursor: + srcname = row[5].rsplit('-', 2)[0] + filename = { + 'rpmname': f"{row[0]}-{row[1]}-{row[3]}.{row[4]}.rpm", + 'name': row[0], + 'version': row[1], + 'epoch': row[2], + 'release': row[3], + 'arch': row[4], + 'sourcerpm': row[5], + 'pkgId': row[6], + 'srcname': srcname + } + filename1.append(filename) + + return filename1 + + except sqlite3.Error as e: + print(f"数据库操作失败: {e}") + return [] + + finally: + if 'con' in locals(): + con.close() +# def plus(*baseurls): +# """ +# 用于处理多个url,合并处理结果 +# """ +# filename2=[] +# PkgList=[] +# for baseurl in baseurls: +# un_path=Getsqlite(baseurl) +# filename1=Getrpminfo(un_path) +# filename2.extend(filename1) +# for filenme in filename2: +# PkgList.append(filenme["rpmname"]) +# print('\n' + '##### 通过 Getrpminfo , 获取软件包nvr 完成：', len(filename2)) # 检修口 +# return PkgList,filename2 +# def plus(*baseurls): +# """ +# 用于处理多个url,合并处理结果 +# """ +# filename2 = [] +# PkgList = [] +# for baseurl in baseurls: +# try: +# un_path = Getsqlite(baseurl) +# filename1 = Getrpminfo(un_path) +# filename2.extend(filename1) +# except Exception as e: +# print(f"处理仓库 {baseurl} 失败: {e}") +# continue # 跳过当前仓库，继续处理下一个 +# +# for filenme in filename2: +# PkgList.append(filenme["rpmname"]) +# +# print(f"成功处理 {len(filename2)} 个软件包") +# return PkgList, filename2 +def plus(*baseurls): + """ + 处理多个仓库 URL，合并结果 + """ + filename2 = [] + PkgList = [] + for baseurl in baseurls: + try: + un_path = Getsqlite(baseurl) + filename1 = Getrpminfo(un_path) + filename2.extend(filename1) + except Exception as e: + print(f"跳过仓库 {baseurl}: {str(e)}") + continue + + for filenme in filename2: + PkgList.append(filenme["rpmname"]) + + print(f"成功处理 {len(filename2)} 个软件包") + return PkgList, filename2 + +def plus1(url): + """ + 用于处理多个url,合并处理结果 + """ + filename2=[] + PkgList=[] + baseurls=url.split(',') + for baseurl in baseurls: + un_path=Getsqlite(baseurl) + # filename1=Getrpminfo(un_path) + filename1 = Getrpminfo(un_path) + filename2.extend(filename1) + for filenme in filename2: + PkgList.append(filenme["rpmname"]) + print('\n' + '##### 通过 Getrpminfo , 获取软件包nvr 完成：', len(filename2)) # 检修口 + return PkgList,filename2 + + +def getsrc(pkglist,filename): + """ + 用于获取对应软件包的源码包名称，name为全名 + """ + srclist=[] + pkgidlist = [] + for pkg in pkglist: + pkg=list(pkg.split()) + for i in filename: + if pkg[-1]==i["rpmname"]: + srclist.append(i["sourcerpm"]) ####获取其他标签### + # pkgidlist.append(i["pkgId"]) ####获取其他标签### + #srclist = list(set(srclist)) + + return srclist + +def getsrc1(pkglist,filename): + """ + 用于获取对应软件包的源码包名称，name为全名 + """ + srclist=[] + pkgidlist = [] + for pkg in pkglist: + pkg=list(pkg.split()) + for i in filename: + if pkg[-1]==i["rpmname"]: + srclist.append(i["sourcerpm"]) ####获取其他标签### + pkgidlist.append(i["pkgId"]) ####获取其他标签### + #srclist = list(set(srclist)) + + return pkglist,pkgidlist,srclist + +def getsrc2(pkglist,filename): + """ + 用于获取对应软件包的源码包名称，name为全名 + """ + srclist=[] + srcnamelist = [] + for pkg in pkglist: + pkg=list(pkg.split()) + for i in filename: + if pkg[-1]==i["rpmname"]: + srclist.append(i["sourcerpm"]) ####获取其他标签### + srcnamelist.append(i["srcname"]) ####获取其他标签### + #srclist = list(set(srclist)) + + return srclist,srcnamelist + +def getpkgid(pkglist,filename): + """ + 用于获取对应软件包的 pkgid，name为全名 + """ + pkgidlist=[] + for pkg in pkglist: + #pkg=list(pkg.split()) + for i in filename: + if pkg==i["rpmname"]: + pkgidlist.append(i["pkgId"]) ####获取其他标签### + #srclist = list(set(srclist)) + #print(pkgidlist) + return pkgidlist + + +def pkgiddiff(pkglist,filename1,filename2): + result1=[] + for pkg in pkglist: + for i in filename1: + if i["rpmname"]==pkg: + pkgId1=i["pkgId"] + for J in filename2: + if J["rpmname"] == pkg: + pkgId2 = J["pkgId"] + if pkgId1 == pkgId2: + result1.append("一致") + else: + result1.append("不一致") + return result1 + + + + +#方法3.0：保存结果，生成txt +def GenTxT(Sourcerpm): + filename = input("please input your filename for generate result: ") + f=open("./"+filename+".txt", "w", encoding="utf-8") + for pkg in Sourcerpm: + f.write(str(pkg+'\n')) + f.close + + +import pandas as pd + + +def write_to_csv(pkglist, pkgidlist, srclist, filename='output.csv'): + """ + 将 pkglist、pkgidlist 和 srclist 写入 CSV 文件。 + + 参数： + pkglist : list + 软件包全名的列表。 + pkgidlist : list + 包 ID 的列表。 + srclist : list + 源码包名称的列表。 + filename : str + 输出 CSV 文件的名称。 + """ + # 创建一个 DataFrame + data = { + 'Package Name': pkglist, + 'Package ID': pkgidlist, + 'Source RPM': srclist + } + df = pd.DataFrame(data) + + # 将 DataFrame 写入 CSV 文件 + df.to_csv(filename, index=False, encoding='utf-8-sig') + +# +# pkglist,filename=plus("https://update.cs2c.com.cn/NS/V10/V10SP3-2403/os/adv/lic/updates/x86_64/") +# print(pkglist,filename) +# pkglist, pkgidlist, srclist=getsrc1(pkglist,filename) + +# write_to_csv(pkglist, pkgidlist, srclist, filename='sp3-loongarch-output.csv') + + +# Getsqlite("https://dl.fedoraproject.org/pub/archive/fedora/linux/releases/36/Server/source/tree/") +# plus("http://10.44.34.84/ns8.2/x86/BaseOS/","http://10.44.34.84/ns8.2/x86/AppStream/") +# plus("https://10.44.16.185/NS/V10/8U8/os/adv/lic/AppStream/x86_64/os/","https://10.44.16.185/NS/V10/8U8/os/adv/lic/BaseOS/x86_64/os/") \ No newline at end of file diff --git a/script/10_black_check/README.md b/script/10_black_check/README.md new file mode 100644 index 0000000..c7292a6 --- /dev/null +++ b/script/10_black_check/README.md @@ -0,0 +1,18 @@ + +## 脚本简介 +本脚本用于测试updates仓库中是否有黑名单列表中的软件包。 +如果有，列出导出的黑名单软件包 +如果没有，则表示测试通过 + +## 使用方法 + 执行脚本：python3 check_black_rpms_in_repos.py V10SP1.1 V10SP2 + 参数V10SP1.1 V10SP2为测试的产品列表，可以有多个，如果没有参数，默认测试所有产品 + 参数的值有：'V10SP1.1', 'V10SP2', 'V10SP3', 'V10SP3-2403', 'V10-HPC', 'V10-ZJ', 'HOST', 'HOST-2309', 'HOST-2406', 'V10SP3-2309A', 'V10SP3-2309B' + + +## 输出: + 测试结果为输出字符串 + 测试通过：说明无黑名单软件包导出到仓库 + 测试有报错：说明有黑名单软件包导出到仓库 + +# 注意事项 diff --git a/script/10_black_check/check_black_rpms_in_repos.py b/script/10_black_check/check_black_rpms_in_repos.py new file mode 100644 index 0000000..11e28b5 --- /dev/null +++ b/script/10_black_check/check_black_rpms_in_repos.py @@ -0,0 +1,119 @@ +import requests +from bs4 import BeautifulSoup +import re +import json +import sys +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + + +def create_retry_session(): + session = requests.Session() + retry = Retry( + total=3, + backoff_factor=1, + status_forcelist=[500, 502, 503, 504] + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + return session + + +def fetch_rpm_packages(url): + session = create_retry_session() + try: + # 发送 HTTP 请求获取仓库页面内容 + response = requests.get(url, timeout=(10, 30)) # 添加超时设置 + response.raise_for_status() + + # 使用 BeautifulSoup 解析 HTML 内容 + soup = BeautifulSoup(response.text, 'html.parser') + + # 使用正则表达式匹配 RPM 包文件名 + rpm_pattern = re.compile(r'.*\.rpm$') + rpm_links = soup.find_all('a', href=rpm_pattern) + + # 提取 RPM 包名称 + packages = [] + for link in rpm_links: + rpm = link.get('href').replace('%2B', '+') # 替换转移字符+ + if rpm.strip(): + rpm_parts = rpm.rsplit("-", 2) # 更清晰的变量名 + if len(rpm_parts) >= 3: # 确保至少分割出包名、版本和架构 + name = rpm_parts[0] + packages.append(name) + else: + print(f"无法解析的软件包: {rpm}") + + # 按包名排序并返回 + packages.sort() + print(f"从仓库获取到软件包列表，共 {len(packages)} 个 RPM 包") + return packages + except requests.exceptions.RequestException as e: + print(f"获取RPM包列表时出错: {e}") + return [] # 失败时返回空列表 + except Exception as e: + print(f"获取RPM包列表时发生未知错误: {e}") + return [] + + +def fetch_and_read_remote_file(url): + try: + # 发送 HTTP 请求获取文件内容，设置超时 + response = requests.get(url, timeout=10) + response.raise_for_status() + + # 获取文件内容并按行分割 + lines = response.text.splitlines() + lines.sort() # 排序 + print(f"成功获取黑名单列表，共 {len(lines)} 个 RPM 包") + return lines + except requests.exceptions.HTTPError as http_err: + print(f"黑名单获取：HTTP 错误发生: {http_err}") + return [] + except requests.exceptions.RequestException as req_err: + print(f"黑名单获取：请求错误发生: {req_err}") + return [] + except Exception as err: + print(f"黑名单获取：其他错误发生: {err}") + return [] + + +if __name__ == "__main__": + with open('repository_config.json', 'r', encoding='utf-8') as file: + data = json.load(file) + if len(sys.argv) > 1: + print("测试产品:", sys.argv[1:]) # 跳过脚本名 + product_list = sys.argv[1:] + print(product_list) + else: + print("未提供具体测试产品,默认测试所有产品") + product_list = list(data.keys()) + print(f"可选其中的值作为脚本参数：{product_list}") + # 解析json文件 + for product in product_list: + try: + print(f"\n--------产品【{product}】--------") + product_data = data.get(product, {}) + if not product_data: + print(f"警告：未找到产品 {product} 的配置") + continue + + for arch, urls in product_data.items(): + print(f"\n{arch}仓库：") + if len(urls) < 2: + print(f"警告：{product}/{arch} 配置不完整，跳过") + continue + + rpm_packages = fetch_rpm_packages(urls[0]) + rpm_black_packages = fetch_and_read_remote_file(urls[1]) + + set_common = set(rpm_packages) & set(rpm_black_packages) + + if set_common: + print(f"测试有报错：发现 {len(set_common)} 个黑名单软件包：{set_common}") + else: + print("测试通过：updates仓库没有发现黑名单软件包") + except Exception as err: + print(f"{product} 仓库处理失败：{err}") diff --git a/script/10_black_check/repository_config.json b/script/10_black_check/repository_config.json new file mode 100644 index 0000000..1dc0a8d --- /dev/null +++ b/script/10_black_check/repository_config.json @@ -0,0 +1,124 @@ +{ + "V10SP1.1": { + "x86_64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/updates/x86_64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/V10SP1.1/os/adv/lic/updates/x86_64/black" + ], + "aarch64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/updates/aarch64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/V10SP1.1/os/adv/lic/updates/aarch64/black" + ], + "loongarch64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/updates/loongarch64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/V10SP1.1/os/adv/lic/updates/loongarch64/black" + ], + "mips64el": [ + "https://update.cs2c.com.cn/private_test/repo/V10/V10SP1.1/os/adv/lic/updates/mips64el/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/V10SP1.1/os/adv/lic/updates/mips64el/black" + ] + }, + "V10SP2": { + "x86_64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/V10SP2/os/adv/lic/updates/x86_64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/V10SP2/os/adv/lic/updates/x86_64/black" + ], + "aarch64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/V10SP2/os/adv/lic/updates/aarch64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/V10SP2/os/adv/lic/updates/aarch64/black" + ] + }, + "V10SP3": { + "x86_64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/V10SP3/os/adv/lic/updates/x86_64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/V10SP3/os/adv/lic/updates/x86_64/black" + ], + "aarch64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/V10SP3/os/adv/lic/updates/aarch64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/V10SP3/os/adv/lic/updates/aarch64/black" + ], + "loongarch64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/V10SP3/os/adv/lic/updates/loongarch64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/V10SP3/os/adv/lic/updates/loongarch64/black" + ] + }, + "V10SP3-2403": { + "x86_64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/x86_64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/V10SP3-2403/os/adv/lic/updates/x86_64/black" + ], + "aarch64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/aarch64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/V10SP3-2403/os/adv/lic/updates/aarch64/black" + ], + "loongarch64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/V10SP3-2403/os/adv/lic/updates/loongarch64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/V10SP3-2403/os/adv/lic/updates/loongarch64/black" + ] + }, + "V10-HPC": { + "x86_64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/HPC/os/adv/lic/updates/x86_64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/HPC/os/adv/lic/updates/x86_64/black" + ], + "aarch64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/HPC/os/adv/lic/updates/aarch64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/HPC/os/adv/lic/updates/aarch64/black" + ] + }, + "V10-ZJ": { + "x86_64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/V10-ZJ/os/adv/lic/updates/x86_64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/V10-ZJ/os/adv/lic/updates/x86_64/black" + ], + "aarch64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/V10-ZJ/os/adv/lic/updates/aarch64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/V10-ZJ/os/adv/lic/updates/aarch64/black" + ], + "mips64el": [ + "https://update.cs2c.com.cn/private_test/repo/V10/V10-ZJ/os/adv/lic/updates/mips64el/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/V10-ZJ/os/adv/lic/updates/mips64el/black" + ] + }, + "HOST": { + "x86_64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/HOST/SP3/os/adv/lic/updates/x86_64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/HOST/SP3/adv/lic/updates/x86_64/black" + ], + "aarch64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/HOST/SP3/os/adv/lic/updates/aarch64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/HOST/SP3/adv/lic/updates/aarch64/black" + ] + }, + "HOST-2309": { + "x86_64": [ + "https://update.cs2c.com.cn/private_test/repo/HOST/2309/os/adv/lic/updates/x86_64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/HOST/2309/os/adv/lic/updates/x86_64/black" + ], + "aarch64": [ + "https://update.cs2c.com.cn/private_test/repo/HOST/2309/os/adv/lic/updates/aarch64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/HOST/2309/os/adv/lic/updates/aarch64/black" + ] + }, + "HOST-2406": { + "x86_64": [ + "https://update.cs2c.com.cn/private_test/repo/HOST/2406/os/adv/lic/updates/x86_64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/HOST/2406/os/adv/lic/updates/x86_64/black" + ], + "aarch64": [ + "https://update.cs2c.com.cn/private_test/repo/HOST/2309/os/adv/lic/updates/aarch64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/HOST/2309/os/adv/lic/updates/aarch64/black" + ] + }, + "V10SP3-2309A": { + "aarch64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/2309A/os/adv/lic/updates/aarch64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/2309A/os/adv/lic/updates/aarch64/black" + ] + }, + "V10SP3-2309B": { + "aarch64": [ + "https://update.cs2c.com.cn/private_test/repo/V10/2309B/os/adv/lic/updates/aarch64/Packages/", + "https://update.cs2c.com.cn/private_test/repo_black/V10/2309B/os/adv/lic/updates/aarch64/black" + ] + } +} \ No newline at end of file