#!/usr/bin/env python3 import os import re import sys import subprocess from pathlib import Path from tempfile import NamedTemporaryFile # Matches e.g. "| [`bash`](bash/) | GNU Bash | 5.0 | https://www.gnu.org/software/bash/ |" # and captures "bash" in group 1, "bash/" in group 2, "" in group 3, "GNU Bash" in group 4, "5.0" in group 5 # and "https://www.gnu.org/software/bash/" in group 6. PORT_TABLE_REGEX = re.compile( r'^\| \[`([^`]+)`\]\(([^\)]+)\)([^\|]+) \| ([^\|]+) \| ([^\|]+?) \| ([^\|]+) \|+$', re.MULTILINE ) # Matches non-abbreviated git hashes GIT_HASH_REGEX = re.compile(r'^[0-9a-f]{40}$') PORT_TABLE_FILE = 'AvailablePorts.md' IGNORE_FILES = { '.gitignore', '.port_include.sh', '.strip_env.sh', PORT_TABLE_FILE, 'build_all.sh', 'build_installed.sh', 'README.md', '.hosted_defs.sh' } # Matches port names in Ports/foo/ReadMe.md PORT_NAME_REGEX = re.compile(r'([ .()[\]{}\w-]+)\.patch') REQUIRE_GIT_PATCHES = True GIT_PATCH_SUBJECT_RE = re.compile(r'Subject: (.*)\n') def read_port_table(filename): """Open a file and find all PORT_TABLE_REGEX matches. Args: filename (str): filename Returns: set: all PORT_TABLE_REGEX matches """ ports = {} with open(filename, 'r') as fp: matches = PORT_TABLE_REGEX.findall(fp.read()) for match in matches: line_len = sum([len(part) for part in match]) ports[match[0]] = { "dir_ref": match[1], "name": match[2].strip(), "version": match[4].strip(), "url": match[5].strip(), "line_len": line_len } return ports def read_port_dirs(): """Check Ports directory for unexpected files and check each port has a package.sh file. Returns: list: all ports (set), no errors encountered (bool) """ ports = {} all_good = True for entry in os.listdir(): if entry in IGNORE_FILES: continue if not os.path.isdir(entry): print(f"Ports/{entry} is neither a port (not a directory) nor an ignored file?!") all_good = False continue if not os.path.exists(entry + '/package.sh'): print(f"Ports/{entry}/ is missing its package.sh?!") all_good = False continue ports[entry] = get_port_properties(entry) return ports, all_good PORT_PROPERTIES = ('port', 'version', 'files', 'auth_type') def get_port_properties(port): """Retrieves common port properties from its package.sh file. Returns: dict: keys are values from PORT_PROPERTIES, values are from the package.sh file """ package_sh_command = f"./package.sh showproperty {' '.join(PORT_PROPERTIES)}" res = subprocess.run(f"cd {port}; exec {package_sh_command}", shell=True, capture_output=True) if res.returncode == 0: results = res.stdout.decode('utf-8').split('\n\n') props = {prop: results[i].strip() for i, prop in enumerate(PORT_PROPERTIES)} else: print(( f'Executing "{package_sh_command}" script for port {port} failed with ' f'exit code {res.returncode}, output from stderr:\n{res.stderr.decode("utf-8").strip()}' )) props = {x: '' for x in PORT_PROPERTIES} return props def check_package_files(ports): """Check port package.sh file for required properties. Args: ports (list): List of all ports to check Returns: bool: no errors encountered """ all_good = True for port in ports.keys(): package_file = f"{port}/package.sh" if not os.path.exists(package_file): continue props = ports[port] if props['port'] != port: print(f"Ports/{port} should use '{port}' for 'port' but is using '{props['port']}' instead") all_good = False if not props['auth_type'] in ('sha256', 'sig', ''): print(f"Ports/{port} uses invalid signature algorithm '{props['auth_type']}' for 'auth_type'") all_good = False for prop in PORT_PROPERTIES: if prop == 'auth_type' and re.match('^https://github.com/SerenityPorts/', props["files"]): continue if props[prop] == '': print(f"Ports/{port} is missing required property '{prop}'") all_good = False return all_good def get_and_check_port_patch_list(ports): """Checks all port patches and returns the port list/properties Args: ports (list): List of all ports to check Returns: all_good (bool): No errors encountered all_properties (dict): Mapping of port to port properties """ all_port_properties = {} all_good = True for port in ports: patches_directory = f"{port}/patches" if not os.path.exists(patches_directory): continue if not os.path.isdir(patches_directory): print(f"Ports/{port}/patches exists, but is not a directory. This is not right!") all_good = False continue patches_path = Path(patches_directory) patches_readme_path = patches_path / "ReadMe.md" patch_files = set(patches_path.glob("*.patch")) non_patch_files = set(patches_path.glob("*")) - patch_files - {patches_readme_path} port_properties = { "patches_path": patches_path, "patches_readme_path": patches_readme_path, "patch_files": patch_files, "non_patch_files": non_patch_files } all_port_properties[port] = port_properties if len(non_patch_files) != 0: print(f"Ports/{port}/patches contains the following non-patch files:", ', '.join(x.name for x in non_patch_files)) all_good = False return all_good, all_port_properties def check_descriptions_for_port_patches(patches): """Ensure that ports containing patches have them documented. Args: patches (dict): Dictionary mapping ports to all their patches Returns: bool: no errors encountered """ all_good = True for port, properties in patches.items(): patches_readme_path = properties["patches_readme_path"] patch_files = properties["patch_files"] readme_file_exists = patches_readme_path.exists() if len(patch_files) == 0: print(f"Ports/{port}/patches exists, but contains no patches", end="") if readme_file_exists: print(", yet it contains a ReadMe.md") else: print() all_good = False continue if not readme_file_exists: print(f"Ports/{port}/patches contains patches but no ReadMe.md describing them") all_good = False continue with open(str(patches_readme_path), 'r', encoding='utf-8') as f: readme_contents = [] for line in f: if not line.startswith('#'): continue match = PORT_NAME_REGEX.search(line) if match: readme_contents.append(match.group(1)) patch_names = set(Path(x).stem for x in patch_files) for patch_name in patch_names: if patch_name not in readme_contents: print(f"Ports/{port}/patches/{patch_name}.patch does not appear to be described in" " the corresponding ReadMe.md") all_good = False for patch_name in readme_contents: if patch_name not in patch_names: print(f"Ports/{port}/patches/{patch_name}.patch is described in ReadMe.md, " "but does not actually exist") all_good = False return all_good def try_parse_git_patch(path_to_patch): with open(path_to_patch, 'rb') as f: contents_of_patch = f.read() with NamedTemporaryFile('r+b') as message_file: res = subprocess.run( f"git mailinfo {message_file.name} /dev/null", shell=True, capture_output=True, input=contents_of_patch) if res.returncode != 0: return None message = message_file.read().decode('utf-8') subject = GIT_PATCH_SUBJECT_RE.search(res.stdout.decode("utf-8")) if subject: message = subject.group(1) + "\n" + message return message def check_patches_are_git_patches(patches): """Ensure that all patches are patches made by (or compatible with) `git format-patch`. Args: patches (dict): Dictionary mapping ports to all their patches Returns: bool: no errors encountered """ all_good = True for port, properties in patches.items(): for patch_path in properties["patch_files"]: result = try_parse_git_patch(patch_path) if not result: print(f"Ports/{port}/patches: {patch_path.stem} does not appear to be a valid " "git patch.") all_good = False continue return all_good def check_available_ports(from_table, ports): """Check AvailablePorts.md for correct properties. Args: from_table (dict): Ports table from AvailablePorts.md ports (dict): Dictionary with port properties from package.sh Returns: bool: no errors encountered """ all_good = True previous_line_len = None for port in from_table.keys(): if previous_line_len is None: previous_line_len = from_table[port]["line_len"] if previous_line_len != from_table[port]["line_len"]: print(f"Table row for port {port} is improperly aligned with other rows.") all_good = False else: previous_line_len = from_table[port]["line_len"] actual_ref = from_table[port]["dir_ref"] expected_ref = f"{port}/" if actual_ref != expected_ref: print(( f'Directory link target in AvailablePorts.md for port {port} is ' f'incorrect, expected "{expected_ref}", found "{actual_ref}"' )) all_good = False actual_version = from_table[port]["version"] expected_version = ports[port]["version"] if GIT_HASH_REGEX.match(expected_version): expected_version = expected_version[0:7] if expected_version == "git": expected_version = "" if actual_version != expected_version: print(( f'Version in AvailablePorts.md for port {port} is incorrect, ' f'expected "{expected_version}", found "{actual_version}"' )) all_good = False return all_good def run(): """Check Ports directory and package files for errors.""" from_table = read_port_table(PORT_TABLE_FILE) ports, all_good = read_port_dirs() from_table_set = set(from_table.keys()) ports_set = set(ports.keys()) if from_table_set - ports_set: all_good = False print('AvailablePorts.md lists ports that do not appear in the file system:') for port in sorted(from_table_set - ports_set): print(f" {port}") if ports_set - from_table_set: all_good = False print('AvailablePorts.md is missing the following ports:') for port in sorted(ports_set - from_table_set): print(f" {port}") if not check_package_files(ports): all_good = False if not check_available_ports(from_table, ports): all_good = False patch_list_good, port_properties = get_and_check_port_patch_list(ports.keys()) all_good = all_good and patch_list_good if not check_descriptions_for_port_patches(port_properties): all_good = False if REQUIRE_GIT_PATCHES and not check_patches_are_git_patches(port_properties): all_good = False if not all_good: sys.exit(1) print('No issues found.') if __name__ == '__main__': os.chdir(f"{os.path.dirname(__file__)}/../Ports") run()